##// END OF EJS Templates
metrics: allow indexing items with dates in past if permanent storage is enabled
ergo -
Show More
@@ -1,752 +1,765 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2
2
3 # Copyright (C) 2010-2016 RhodeCode GmbH
3 # Copyright (C) 2010-2016 RhodeCode GmbH
4 #
4 #
5 # This program is free software: you can redistribute it and/or modify
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU Affero General Public License, version 3
6 # it under the terms of the GNU Affero General Public License, version 3
7 # (only), as published by the Free Software Foundation.
7 # (only), as published by the Free Software Foundation.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU Affero General Public License
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 #
16 #
17 # This program is dual-licensed. If you wish to learn more about the
17 # This program is dual-licensed. If you wish to learn more about the
18 # AppEnlight Enterprise Edition, including its added features, Support
18 # AppEnlight Enterprise Edition, including its added features, Support
19 # services, and proprietary license terms, please see
19 # services, and proprietary license terms, please see
20 # https://rhodecode.com/licenses/
20 # https://rhodecode.com/licenses/
21
21
22 import datetime
22 import datetime
23
23
24 import colander
24 import colander
25 from colander import null
25 from colander import null
26
26
27 # those keywords are here so we can distingush between searching for tags and
27 # those keywords are here so we can distingush between searching for tags and
28 # normal properties of reports/logs
28 # normal properties of reports/logs
29 accepted_search_params = ['resource',
29 accepted_search_params = ['resource',
30 'request_id',
30 'request_id',
31 'start_date',
31 'start_date',
32 'end_date',
32 'end_date',
33 'page',
33 'page',
34 'min_occurences',
34 'min_occurences',
35 'http_status',
35 'http_status',
36 'priority',
36 'priority',
37 'error',
37 'error',
38 'url_path',
38 'url_path',
39 'url_domain',
39 'url_domain',
40 'report_status',
40 'report_status',
41 'min_duration',
41 'min_duration',
42 'max_duration',
42 'max_duration',
43 'message',
43 'message',
44 'level',
44 'level',
45 'namespace']
45 'namespace']
46
46
47
47
48 @colander.deferred
48 @colander.deferred
49 def deferred_utcnow(node, kw):
49 def deferred_utcnow(node, kw):
50 return kw['utcnow']
50 return kw['utcnow']
51
51
52
52
53 def lowercase_preparer(input_data):
53 def lowercase_preparer(input_data):
54 """
54 """
55 Transforms a list of string entries to lowercase
55 Transforms a list of string entries to lowercase
56 Used in search query validation
56 Used in search query validation
57 """
57 """
58 if not input_data:
58 if not input_data:
59 return input_data
59 return input_data
60 return [x.lower() for x in input_data]
60 return [x.lower() for x in input_data]
61
61
62
62
63 def shortener_factory(cutoff_size=32):
63 def shortener_factory(cutoff_size=32):
64 """
64 """
65 Limits the input data to specific character count
65 Limits the input data to specific character count
66 :arg cutoff_cutoff_size How much characters to store
66 :arg cutoff_cutoff_size How much characters to store
67
67
68 """
68 """
69
69
70 def shortener(input_data):
70 def shortener(input_data):
71 if not input_data:
71 if not input_data:
72 return input_data
72 return input_data
73 else:
73 else:
74 if isinstance(input_data, str):
74 if isinstance(input_data, str):
75 return input_data[:cutoff_size]
75 return input_data[:cutoff_size]
76 else:
76 else:
77 return input_data
77 return input_data
78
78
79 return shortener
79 return shortener
80
80
81
81
82 def cast_to_unicode_or_null(value):
82 def cast_to_unicode_or_null(value):
83 if value is not colander.null:
83 if value is not colander.null:
84 return str(value)
84 return str(value)
85 return None
85 return None
86
86
87
87
88 class NonTZDate(colander.DateTime):
88 class NonTZDate(colander.DateTime):
89 """ Returns null for incorrect date format - also removes tz info"""
89 """ Returns null for incorrect date format - also removes tz info"""
90
90
91 def deserialize(self, node, cstruct):
91 def deserialize(self, node, cstruct):
92 # disabled for now
92 # disabled for now
93 # if cstruct and isinstance(cstruct, str):
93 # if cstruct and isinstance(cstruct, str):
94 # if ':' not in cstruct:
94 # if ':' not in cstruct:
95 # cstruct += ':0.0'
95 # cstruct += ':0.0'
96 # if '.' not in cstruct:
96 # if '.' not in cstruct:
97 # cstruct += '.0'
97 # cstruct += '.0'
98 value = super(NonTZDate, self).deserialize(node, cstruct)
98 value = super(NonTZDate, self).deserialize(node, cstruct)
99 if value:
99 if value:
100 return value.replace(tzinfo=None)
100 return value.replace(tzinfo=None)
101 return value
101 return value
102
102
103
103
104 class UnknownType(object):
104 class UnknownType(object):
105 """
105 """
106 Universal type that will accept a deserialized JSON object and store it unaltered
106 Universal type that will accept a deserialized JSON object and store it unaltered
107 """
107 """
108
108
109 def serialize(self, node, appstruct):
109 def serialize(self, node, appstruct):
110 if appstruct is null:
110 if appstruct is null:
111 return null
111 return null
112 return appstruct
112 return appstruct
113
113
114 def deserialize(self, node, cstruct):
114 def deserialize(self, node, cstruct):
115 if cstruct is null:
115 if cstruct is null:
116 return null
116 return null
117 return cstruct
117 return cstruct
118
118
119 def cstruct_children(self):
119 def cstruct_children(self):
120 return []
120 return []
121
121
122
122
123 # SLOW REPORT SCHEMA
123 # SLOW REPORT SCHEMA
124
124
125 def rewrite_type(input_data):
125 def rewrite_type(input_data):
126 """
126 """
127 Fix for legacy appenlight clients
127 Fix for legacy appenlight clients
128 """
128 """
129 if input_data == 'remote_call':
129 if input_data == 'remote_call':
130 return 'remote'
130 return 'remote'
131 return input_data
131 return input_data
132
132
133
133
134 class ExtraTupleSchema(colander.TupleSchema):
134 class ExtraTupleSchema(colander.TupleSchema):
135 name = colander.SchemaNode(colander.String(),
135 name = colander.SchemaNode(colander.String(),
136 validator=colander.Length(1, 64))
136 validator=colander.Length(1, 64))
137 value = colander.SchemaNode(UnknownType(),
137 value = colander.SchemaNode(UnknownType(),
138 preparer=shortener_factory(512),
138 preparer=shortener_factory(512),
139 missing=None)
139 missing=None)
140
140
141
141
142 class ExtraSchemaList(colander.SequenceSchema):
142 class ExtraSchemaList(colander.SequenceSchema):
143 tag = ExtraTupleSchema()
143 tag = ExtraTupleSchema()
144 missing = None
144 missing = None
145
145
146
146
147 class TagsTupleSchema(colander.TupleSchema):
147 class TagsTupleSchema(colander.TupleSchema):
148 name = colander.SchemaNode(colander.String(),
148 name = colander.SchemaNode(colander.String(),
149 validator=colander.Length(1, 128))
149 validator=colander.Length(1, 128))
150 value = colander.SchemaNode(UnknownType(),
150 value = colander.SchemaNode(UnknownType(),
151 preparer=shortener_factory(128),
151 preparer=shortener_factory(128),
152 missing=None)
152 missing=None)
153
153
154
154
155 class TagSchemaList(colander.SequenceSchema):
155 class TagSchemaList(colander.SequenceSchema):
156 tag = TagsTupleSchema()
156 tag = TagsTupleSchema()
157 missing = None
157 missing = None
158
158
159
159
160 class NumericTagsTupleSchema(colander.TupleSchema):
160 class NumericTagsTupleSchema(colander.TupleSchema):
161 name = colander.SchemaNode(colander.String(),
161 name = colander.SchemaNode(colander.String(),
162 validator=colander.Length(1, 128))
162 validator=colander.Length(1, 128))
163 value = colander.SchemaNode(colander.Float(), missing=0)
163 value = colander.SchemaNode(colander.Float(), missing=0)
164
164
165
165
166 class NumericTagSchemaList(colander.SequenceSchema):
166 class NumericTagSchemaList(colander.SequenceSchema):
167 tag = NumericTagsTupleSchema()
167 tag = NumericTagsTupleSchema()
168 missing = None
168 missing = None
169
169
170
170
171 class SlowCallSchema(colander.MappingSchema):
171 class SlowCallSchema(colander.MappingSchema):
172 """
172 """
173 Validates slow call format in slow call list
173 Validates slow call format in slow call list
174 """
174 """
175 start = colander.SchemaNode(NonTZDate())
175 start = colander.SchemaNode(NonTZDate())
176 end = colander.SchemaNode(NonTZDate())
176 end = colander.SchemaNode(NonTZDate())
177 statement = colander.SchemaNode(colander.String(), missing='')
177 statement = colander.SchemaNode(colander.String(), missing='')
178 parameters = colander.SchemaNode(UnknownType(), missing=None)
178 parameters = colander.SchemaNode(UnknownType(), missing=None)
179 type = colander.SchemaNode(
179 type = colander.SchemaNode(
180 colander.String(),
180 colander.String(),
181 preparer=rewrite_type,
181 preparer=rewrite_type,
182 validator=colander.OneOf(
182 validator=colander.OneOf(
183 ['tmpl', 'sql', 'nosql', 'remote', 'unknown', 'custom']),
183 ['tmpl', 'sql', 'nosql', 'remote', 'unknown', 'custom']),
184 missing='unknown')
184 missing='unknown')
185 subtype = colander.SchemaNode(colander.String(),
185 subtype = colander.SchemaNode(colander.String(),
186 validator=colander.Length(1, 16),
186 validator=colander.Length(1, 16),
187 missing='unknown')
187 missing='unknown')
188 location = colander.SchemaNode(colander.String(),
188 location = colander.SchemaNode(colander.String(),
189 validator=colander.Length(1, 255),
189 validator=colander.Length(1, 255),
190 missing='')
190 missing='')
191
191
192
192
193 def limited_date(node, value):
193 def limited_date(node, value):
194 """ checks to make sure that the value is not older/newer than 2h """
194 """ checks to make sure that the value is not older/newer than 2h """
195 past_hours = 72
195 past_hours = 72
196 future_hours = 2
196 future_hours = 2
197 min_time = datetime.datetime.utcnow() - datetime.timedelta(
197 min_time = datetime.datetime.utcnow() - datetime.timedelta(
198 hours=past_hours)
198 hours=past_hours)
199 max_time = datetime.datetime.utcnow() + datetime.timedelta(
199 max_time = datetime.datetime.utcnow() + datetime.timedelta(
200 hours=future_hours)
200 hours=future_hours)
201 if min_time > value:
201 if min_time > value:
202 msg = '%r is older from current UTC time by ' + str(past_hours)
202 msg = '%r is older from current UTC time by ' + str(past_hours)
203 msg += ' hours. Ask administrator to enable permanent logging for ' \
203 msg += ' hours. Ask administrator to enable permanent logging for ' \
204 'your application to store logs with dates in past.'
204 'your application to store logs with dates in past.'
205 raise colander.Invalid(node, msg % value)
205 raise colander.Invalid(node, msg % value)
206 if max_time < value:
206 if max_time < value:
207 msg = '%r is newer from current UTC time by ' + str(future_hours)
207 msg = '%r is newer from current UTC time by ' + str(future_hours)
208 msg += ' hours. Ask administrator to enable permanent logging for ' \
208 msg += ' hours. Ask administrator to enable permanent logging for ' \
209 'your application to store logs with dates in future.'
209 'your application to store logs with dates in future.'
210 raise colander.Invalid(node, msg % value)
210 raise colander.Invalid(node, msg % value)
211
211
212
212
213 class SlowCallListSchema(colander.SequenceSchema):
213 class SlowCallListSchema(colander.SequenceSchema):
214 """
214 """
215 Validates list of individual slow calls
215 Validates list of individual slow calls
216 """
216 """
217 slow_call = SlowCallSchema()
217 slow_call = SlowCallSchema()
218
218
219
219
220 class RequestStatsSchema(colander.MappingSchema):
220 class RequestStatsSchema(colander.MappingSchema):
221 """
221 """
222 Validates format of requests statistics dictionary
222 Validates format of requests statistics dictionary
223 """
223 """
224 main = colander.SchemaNode(colander.Float(), validator=colander.Range(0),
224 main = colander.SchemaNode(colander.Float(), validator=colander.Range(0),
225 missing=0)
225 missing=0)
226 sql = colander.SchemaNode(colander.Float(), validator=colander.Range(0),
226 sql = colander.SchemaNode(colander.Float(), validator=colander.Range(0),
227 missing=0)
227 missing=0)
228 nosql = colander.SchemaNode(colander.Float(), validator=colander.Range(0),
228 nosql = colander.SchemaNode(colander.Float(), validator=colander.Range(0),
229 missing=0)
229 missing=0)
230 remote = colander.SchemaNode(colander.Float(), validator=colander.Range(0),
230 remote = colander.SchemaNode(colander.Float(), validator=colander.Range(0),
231 missing=0)
231 missing=0)
232 tmpl = colander.SchemaNode(colander.Float(), validator=colander.Range(0),
232 tmpl = colander.SchemaNode(colander.Float(), validator=colander.Range(0),
233 missing=0)
233 missing=0)
234 custom = colander.SchemaNode(colander.Float(), validator=colander.Range(0),
234 custom = colander.SchemaNode(colander.Float(), validator=colander.Range(0),
235 missing=0)
235 missing=0)
236 sql_calls = colander.SchemaNode(colander.Float(),
236 sql_calls = colander.SchemaNode(colander.Float(),
237 validator=colander.Range(0),
237 validator=colander.Range(0),
238 missing=0)
238 missing=0)
239 nosql_calls = colander.SchemaNode(colander.Float(),
239 nosql_calls = colander.SchemaNode(colander.Float(),
240 validator=colander.Range(0),
240 validator=colander.Range(0),
241 missing=0)
241 missing=0)
242 remote_calls = colander.SchemaNode(colander.Float(),
242 remote_calls = colander.SchemaNode(colander.Float(),
243 validator=colander.Range(0),
243 validator=colander.Range(0),
244 missing=0)
244 missing=0)
245 tmpl_calls = colander.SchemaNode(colander.Float(),
245 tmpl_calls = colander.SchemaNode(colander.Float(),
246 validator=colander.Range(0),
246 validator=colander.Range(0),
247 missing=0)
247 missing=0)
248 custom_calls = colander.SchemaNode(colander.Float(),
248 custom_calls = colander.SchemaNode(colander.Float(),
249 validator=colander.Range(0),
249 validator=colander.Range(0),
250 missing=0)
250 missing=0)
251
251
252
252
253 class FrameInfoVarSchema(colander.SequenceSchema):
253 class FrameInfoVarSchema(colander.SequenceSchema):
254 """
254 """
255 Validates format of frame variables of a traceback
255 Validates format of frame variables of a traceback
256 """
256 """
257 vars = colander.SchemaNode(UnknownType(),
257 vars = colander.SchemaNode(UnknownType(),
258 validator=colander.Length(2, 2))
258 validator=colander.Length(2, 2))
259
259
260
260
261 class FrameInfoSchema(colander.MappingSchema):
261 class FrameInfoSchema(colander.MappingSchema):
262 """
262 """
263 Validates format of a traceback line
263 Validates format of a traceback line
264 """
264 """
265 cline = colander.SchemaNode(colander.String(), missing='')
265 cline = colander.SchemaNode(colander.String(), missing='')
266 module = colander.SchemaNode(colander.String(), missing='')
266 module = colander.SchemaNode(colander.String(), missing='')
267 line = colander.SchemaNode(colander.String(), missing='')
267 line = colander.SchemaNode(colander.String(), missing='')
268 file = colander.SchemaNode(colander.String(), missing='')
268 file = colander.SchemaNode(colander.String(), missing='')
269 fn = colander.SchemaNode(colander.String(), missing='')
269 fn = colander.SchemaNode(colander.String(), missing='')
270 vars = FrameInfoVarSchema()
270 vars = FrameInfoVarSchema()
271
271
272
272
273 class FrameInfoListSchema(colander.SequenceSchema):
273 class FrameInfoListSchema(colander.SequenceSchema):
274 """
274 """
275 Validates format of list of traceback lines
275 Validates format of list of traceback lines
276 """
276 """
277 frame = colander.SchemaNode(UnknownType())
277 frame = colander.SchemaNode(UnknownType())
278
278
279
279
280 class ReportDetailBaseSchema(colander.MappingSchema):
280 class ReportDetailBaseSchema(colander.MappingSchema):
281 """
281 """
282 Validates format of report - ie. request parameters and stats for a request in report group
282 Validates format of report - ie. request parameters and stats for a request in report group
283 """
283 """
284 username = colander.SchemaNode(colander.String(),
284 username = colander.SchemaNode(colander.String(),
285 preparer=[shortener_factory(255),
285 preparer=[shortener_factory(255),
286 lambda x: x or ''],
286 lambda x: x or ''],
287 missing='')
287 missing='')
288 request_id = colander.SchemaNode(colander.String(),
288 request_id = colander.SchemaNode(colander.String(),
289 preparer=shortener_factory(40),
289 preparer=shortener_factory(40),
290 missing='')
290 missing='')
291 url = colander.SchemaNode(colander.String(),
291 url = colander.SchemaNode(colander.String(),
292 preparer=shortener_factory(1024), missing='')
292 preparer=shortener_factory(1024), missing='')
293 ip = colander.SchemaNode(colander.String(), preparer=shortener_factory(39),
293 ip = colander.SchemaNode(colander.String(), preparer=shortener_factory(39),
294 missing=None)
294 missing=None)
295 start_time = colander.SchemaNode(NonTZDate(), validator=limited_date,
295 start_time = colander.SchemaNode(NonTZDate(), validator=limited_date,
296 missing=deferred_utcnow)
296 missing=deferred_utcnow)
297 end_time = colander.SchemaNode(NonTZDate(), validator=limited_date,
297 end_time = colander.SchemaNode(NonTZDate(), validator=limited_date,
298 missing=None)
298 missing=None)
299 user_agent = colander.SchemaNode(colander.String(),
299 user_agent = colander.SchemaNode(colander.String(),
300 preparer=[shortener_factory(512),
300 preparer=[shortener_factory(512),
301 lambda x: x or ''],
301 lambda x: x or ''],
302 missing='')
302 missing='')
303 message = colander.SchemaNode(colander.String(),
303 message = colander.SchemaNode(colander.String(),
304 preparer=shortener_factory(2048),
304 preparer=shortener_factory(2048),
305 missing='')
305 missing='')
306 group_string = colander.SchemaNode(colander.String(),
306 group_string = colander.SchemaNode(colander.String(),
307 validator=colander.Length(1, 512),
307 validator=colander.Length(1, 512),
308 missing=None)
308 missing=None)
309 request_stats = RequestStatsSchema(missing=None)
309 request_stats = RequestStatsSchema(missing=None)
310 request = colander.SchemaNode(colander.Mapping(unknown='preserve'),
310 request = colander.SchemaNode(colander.Mapping(unknown='preserve'),
311 missing={})
311 missing={})
312 traceback = FrameInfoListSchema(missing=None)
312 traceback = FrameInfoListSchema(missing=None)
313 slow_calls = SlowCallListSchema(missing=[])
313 slow_calls = SlowCallListSchema(missing=[])
314 extra = ExtraSchemaList()
314 extra = ExtraSchemaList()
315
315
316
316
317 class ReportDetailSchema_0_5(ReportDetailBaseSchema):
317 class ReportDetailSchema_0_5(ReportDetailBaseSchema):
318 pass
318 pass
319
319
320
320
321 class ReportDetailSchemaPermissiveDate_0_5(ReportDetailSchema_0_5):
321 class ReportDetailSchemaPermissiveDate_0_5(ReportDetailSchema_0_5):
322 start_time = colander.SchemaNode(NonTZDate(), missing=deferred_utcnow)
322 start_time = colander.SchemaNode(NonTZDate(), missing=deferred_utcnow)
323 end_time = colander.SchemaNode(NonTZDate(), missing=None)
323 end_time = colander.SchemaNode(NonTZDate(), missing=None)
324
324
325
325
326 class ReportSchemaBase(colander.MappingSchema):
326 class ReportSchemaBase(colander.MappingSchema):
327 """
327 """
328 Validates format of report group
328 Validates format of report group
329 """
329 """
330 client = colander.SchemaNode(colander.String(),
330 client = colander.SchemaNode(colander.String(),
331 preparer=lambda x: x or 'unknown')
331 preparer=lambda x: x or 'unknown')
332 server = colander.SchemaNode(
332 server = colander.SchemaNode(
333 colander.String(),
333 colander.String(),
334 preparer=[
334 preparer=[
335 lambda x: x.lower() if x else 'unknown', shortener_factory(128)],
335 lambda x: x.lower() if x else 'unknown', shortener_factory(128)],
336 missing='unknown')
336 missing='unknown')
337 priority = colander.SchemaNode(colander.Int(),
337 priority = colander.SchemaNode(colander.Int(),
338 preparer=[lambda x: x or 5],
338 preparer=[lambda x: x or 5],
339 validator=colander.Range(1, 10),
339 validator=colander.Range(1, 10),
340 missing=5)
340 missing=5)
341 language = colander.SchemaNode(colander.String(), missing='unknown')
341 language = colander.SchemaNode(colander.String(), missing='unknown')
342 error = colander.SchemaNode(colander.String(),
342 error = colander.SchemaNode(colander.String(),
343 preparer=shortener_factory(512),
343 preparer=shortener_factory(512),
344 missing='')
344 missing='')
345 view_name = colander.SchemaNode(colander.String(),
345 view_name = colander.SchemaNode(colander.String(),
346 preparer=[shortener_factory(128),
346 preparer=[shortener_factory(128),
347 lambda x: x or ''],
347 lambda x: x or ''],
348 missing='')
348 missing='')
349 http_status = colander.SchemaNode(colander.Int(),
349 http_status = colander.SchemaNode(colander.Int(),
350 preparer=[lambda x: x or 200],
350 preparer=[lambda x: x or 200],
351 validator=colander.Range(1))
351 validator=colander.Range(1))
352
352
353 occurences = colander.SchemaNode(colander.Int(),
353 occurences = colander.SchemaNode(colander.Int(),
354 validator=colander.Range(1, 99999999999),
354 validator=colander.Range(1, 99999999999),
355 missing=1)
355 missing=1)
356 tags = TagSchemaList()
356 tags = TagSchemaList()
357
357
358
358
359 class ReportSchema_0_5(ReportSchemaBase, ReportDetailSchema_0_5):
359 class ReportSchema_0_5(ReportSchemaBase, ReportDetailSchema_0_5):
360 pass
360 pass
361
361
362
362
363 class ReportSchemaPermissiveDate_0_5(ReportSchemaBase,
363 class ReportSchemaPermissiveDate_0_5(ReportSchemaBase,
364 ReportDetailSchemaPermissiveDate_0_5):
364 ReportDetailSchemaPermissiveDate_0_5):
365 pass
365 pass
366
366
367
367
368 class ReportListSchema_0_5(colander.SequenceSchema):
368 class ReportListSchema_0_5(colander.SequenceSchema):
369 """
369 """
370 Validates format of list of report groups
370 Validates format of list of report groups
371 """
371 """
372 report = ReportSchema_0_5()
372 report = ReportSchema_0_5()
373 validator = colander.Length(1)
373 validator = colander.Length(1)
374
374
375
375
376 class ReportListPermissiveDateSchema_0_5(colander.SequenceSchema):
376 class ReportListPermissiveDateSchema_0_5(colander.SequenceSchema):
377 """
377 """
378 Validates format of list of report groups
378 Validates format of list of report groups
379 """
379 """
380 report = ReportSchemaPermissiveDate_0_5()
380 report = ReportSchemaPermissiveDate_0_5()
381 validator = colander.Length(1)
381 validator = colander.Length(1)
382
382
383
383
384 class LogSchema(colander.MappingSchema):
384 class LogSchema(colander.MappingSchema):
385 """
385 """
386 Validates format if individual log entry
386 Validates format if individual log entry
387 """
387 """
388 primary_key = colander.SchemaNode(UnknownType(),
388 primary_key = colander.SchemaNode(UnknownType(),
389 preparer=[cast_to_unicode_or_null,
389 preparer=[cast_to_unicode_or_null,
390 shortener_factory(128)],
390 shortener_factory(128)],
391 missing=None)
391 missing=None)
392 log_level = colander.SchemaNode(colander.String(),
392 log_level = colander.SchemaNode(colander.String(),
393 preparer=shortener_factory(10),
393 preparer=shortener_factory(10),
394 missing='UNKNOWN')
394 missing='UNKNOWN')
395 message = colander.SchemaNode(colander.String(),
395 message = colander.SchemaNode(colander.String(),
396 preparer=shortener_factory(4096),
396 preparer=shortener_factory(4096),
397 missing='')
397 missing='')
398 namespace = colander.SchemaNode(colander.String(),
398 namespace = colander.SchemaNode(colander.String(),
399 preparer=shortener_factory(128),
399 preparer=shortener_factory(128),
400 missing='')
400 missing='')
401 request_id = colander.SchemaNode(colander.String(),
401 request_id = colander.SchemaNode(colander.String(),
402 preparer=shortener_factory(40),
402 preparer=shortener_factory(40),
403 missing='')
403 missing='')
404 server = colander.SchemaNode(colander.String(),
404 server = colander.SchemaNode(colander.String(),
405 preparer=shortener_factory(128),
405 preparer=shortener_factory(128),
406 missing='unknown')
406 missing='unknown')
407 date = colander.SchemaNode(NonTZDate(),
407 date = colander.SchemaNode(NonTZDate(),
408 validator=limited_date,
408 validator=limited_date,
409 missing=deferred_utcnow)
409 missing=deferred_utcnow)
410 tags = TagSchemaList()
410 tags = TagSchemaList()
411
411
412
412
413 class LogSchemaPermanent(LogSchema):
413 class LogSchemaPermanent(LogSchema):
414 date = colander.SchemaNode(NonTZDate(),
414 date = colander.SchemaNode(NonTZDate(),
415 missing=deferred_utcnow)
415 missing=deferred_utcnow)
416 permanent = colander.SchemaNode(colander.Boolean(), missing=False)
416 permanent = colander.SchemaNode(colander.Boolean(), missing=False)
417
417
418
418
419 class LogListSchema(colander.SequenceSchema):
419 class LogListSchema(colander.SequenceSchema):
420 """
420 """
421 Validates format of list of log entries
421 Validates format of list of log entries
422 """
422 """
423 log = LogSchema()
423 log = LogSchema()
424 validator = colander.Length(1)
424 validator = colander.Length(1)
425
425
426
426
427 class LogListPermanentSchema(colander.SequenceSchema):
427 class LogListPermanentSchema(colander.SequenceSchema):
428 """
428 """
429 Validates format of list of log entries
429 Validates format of list of log entries
430 """
430 """
431 log = LogSchemaPermanent()
431 log = LogSchemaPermanent()
432 validator = colander.Length(1)
432 validator = colander.Length(1)
433
433
434
434
435 class ViewRequestStatsSchema(RequestStatsSchema):
435 class ViewRequestStatsSchema(RequestStatsSchema):
436 requests = colander.SchemaNode(colander.Integer(),
436 requests = colander.SchemaNode(colander.Integer(),
437 validator=colander.Range(0),
437 validator=colander.Range(0),
438 missing=0)
438 missing=0)
439
439
440
440
441 class ViewMetricTupleSchema(colander.TupleSchema):
441 class ViewMetricTupleSchema(colander.TupleSchema):
442 """
442 """
443 Validates list of views and their corresponding request stats object ie:
443 Validates list of views and their corresponding request stats object ie:
444 ["dir/module:func",{"custom": 0.0..}]
444 ["dir/module:func",{"custom": 0.0..}]
445 """
445 """
446 view_name = colander.SchemaNode(colander.String(),
446 view_name = colander.SchemaNode(colander.String(),
447 preparer=[shortener_factory(128),
447 preparer=[shortener_factory(128),
448 lambda x: x or 'unknown'],
448 lambda x: x or 'unknown'],
449 missing='unknown')
449 missing='unknown')
450 metrics = ViewRequestStatsSchema()
450 metrics = ViewRequestStatsSchema()
451
451
452
452
453 class ViewMetricListSchema(colander.SequenceSchema):
453 class ViewMetricListSchema(colander.SequenceSchema):
454 """
454 """
455 Validates view breakdown stats objects list
455 Validates view breakdown stats objects list
456 {metrics key of server/time object}
456 {metrics key of server/time object}
457 """
457 """
458 view_tuple = ViewMetricTupleSchema()
458 view_tuple = ViewMetricTupleSchema()
459 validator = colander.Length(1)
459 validator = colander.Length(1)
460
460
461
461
462 class ViewMetricSchema(colander.MappingSchema):
462 class ViewMetricSchema(colander.MappingSchema):
463 """
463 """
464 Validates server/timeinterval object, ie:
464 Validates server/timeinterval object, ie:
465 {server/time object}
465 {server/time object}
466
466
467 """
467 """
468 timestamp = colander.SchemaNode(NonTZDate(),
468 timestamp = colander.SchemaNode(NonTZDate(),
469 validator=limited_date,
469 validator=limited_date,
470 missing=None)
470 missing=None)
471 server = colander.SchemaNode(colander.String(),
471 server = colander.SchemaNode(colander.String(),
472 preparer=[shortener_factory(128),
472 preparer=[shortener_factory(128),
473 lambda x: x or 'unknown'],
473 lambda x: x or 'unknown'],
474 missing='unknown')
474 missing='unknown')
475 metrics = ViewMetricListSchema()
475 metrics = ViewMetricListSchema()
476
476
477
477
478 class GeneralMetricSchema(colander.MappingSchema):
478 class GeneralMetricSchema(colander.MappingSchema):
479 """
479 """
480 Validates universal metric schema
480 Validates universal metric schema
481
481
482 """
482 """
483 namespace = colander.SchemaNode(colander.String(), missing='',
483 namespace = colander.SchemaNode(colander.String(), missing='',
484 preparer=shortener_factory(128))
484 preparer=shortener_factory(128))
485
485
486 server_name = colander.SchemaNode(colander.String(),
486 server_name = colander.SchemaNode(colander.String(),
487 preparer=[shortener_factory(128),
487 preparer=[shortener_factory(128),
488 lambda x: x or 'unknown'],
488 lambda x: x or 'unknown'],
489 missing='unknown')
489 missing='unknown')
490 timestamp = colander.SchemaNode(NonTZDate(), validator=limited_date,
490 timestamp = colander.SchemaNode(NonTZDate(), validator=limited_date,
491 missing=deferred_utcnow)
491 missing=deferred_utcnow)
492 tags = TagSchemaList(missing=colander.required)
492 tags = TagSchemaList(missing=colander.required)
493
493
494
494
495 class GeneralMetricPermanentSchema(GeneralMetricSchema):
496 """
497 Validates universal metric schema
498
499 """
500 timestamp = colander.SchemaNode(NonTZDate(), missing=deferred_utcnow)
501
502
495 class GeneralMetricsListSchema(colander.SequenceSchema):
503 class GeneralMetricsListSchema(colander.SequenceSchema):
496 metric = GeneralMetricSchema()
504 metric = GeneralMetricSchema()
497 validator = colander.Length(1)
505 validator = colander.Length(1)
498
506
499
507
508 class GeneralMetricsPermanentListSchema(colander.SequenceSchema):
509 metric = GeneralMetricPermanentSchema()
510 validator = colander.Length(1)
511
512
500 class MetricsListSchema(colander.SequenceSchema):
513 class MetricsListSchema(colander.SequenceSchema):
501 """
514 """
502 Validates list of metrics objects ie:
515 Validates list of metrics objects ie:
503 [{server/time object}, ] part
516 [{server/time object}, ] part
504
517
505
518
506 """
519 """
507 metric = ViewMetricSchema()
520 metric = ViewMetricSchema()
508 validator = colander.Length(1)
521 validator = colander.Length(1)
509
522
510
523
511 class StringToAppList(object):
524 class StringToAppList(object):
512 """
525 """
513 Returns validated list of application ids from user query and
526 Returns validated list of application ids from user query and
514 set of applications user is allowed to look at
527 set of applications user is allowed to look at
515 transform string to list containing single integer
528 transform string to list containing single integer
516 """
529 """
517
530
518 def serialize(self, node, appstruct):
531 def serialize(self, node, appstruct):
519 if appstruct is null:
532 if appstruct is null:
520 return null
533 return null
521 return appstruct
534 return appstruct
522
535
523 def deserialize(self, node, cstruct):
536 def deserialize(self, node, cstruct):
524 if cstruct is null:
537 if cstruct is null:
525 return null
538 return null
526
539
527 apps = set([int(a) for a in node.bindings['resources']])
540 apps = set([int(a) for a in node.bindings['resources']])
528
541
529 if isinstance(cstruct, str):
542 if isinstance(cstruct, str):
530 cstruct = [cstruct]
543 cstruct = [cstruct]
531
544
532 cstruct = [int(a) for a in cstruct]
545 cstruct = [int(a) for a in cstruct]
533
546
534 valid_apps = list(apps.intersection(set(cstruct)))
547 valid_apps = list(apps.intersection(set(cstruct)))
535 if valid_apps:
548 if valid_apps:
536 return valid_apps
549 return valid_apps
537 return null
550 return null
538
551
539 def cstruct_children(self):
552 def cstruct_children(self):
540 return []
553 return []
541
554
542
555
543 @colander.deferred
556 @colander.deferred
544 def possible_applications_validator(node, kw):
557 def possible_applications_validator(node, kw):
545 possible_apps = [int(a) for a in kw['resources']]
558 possible_apps = [int(a) for a in kw['resources']]
546 return colander.All(colander.ContainsOnly(possible_apps),
559 return colander.All(colander.ContainsOnly(possible_apps),
547 colander.Length(1))
560 colander.Length(1))
548
561
549
562
550 @colander.deferred
563 @colander.deferred
551 def possible_applications(node, kw):
564 def possible_applications(node, kw):
552 return [int(a) for a in kw['resources']]
565 return [int(a) for a in kw['resources']]
553
566
554
567
555 @colander.deferred
568 @colander.deferred
556 def today_start(node, kw):
569 def today_start(node, kw):
557 return datetime.datetime.utcnow().replace(second=0, microsecond=0,
570 return datetime.datetime.utcnow().replace(second=0, microsecond=0,
558 minute=0,
571 minute=0,
559 hour=0)
572 hour=0)
560
573
561
574
562 @colander.deferred
575 @colander.deferred
563 def today_end(node, kw):
576 def today_end(node, kw):
564 return datetime.datetime.utcnow().replace(second=0, microsecond=0,
577 return datetime.datetime.utcnow().replace(second=0, microsecond=0,
565 minute=59, hour=23)
578 minute=59, hour=23)
566
579
567
580
568 @colander.deferred
581 @colander.deferred
569 def old_start(node, kw):
582 def old_start(node, kw):
570 t_delta = datetime.timedelta(days=90)
583 t_delta = datetime.timedelta(days=90)
571 return datetime.datetime.utcnow().replace(second=0, microsecond=0,
584 return datetime.datetime.utcnow().replace(second=0, microsecond=0,
572 minute=0,
585 minute=0,
573 hour=0) - t_delta
586 hour=0) - t_delta
574
587
575
588
576 @colander.deferred
589 @colander.deferred
577 def today_end(node, kw):
590 def today_end(node, kw):
578 return datetime.datetime.utcnow().replace(second=0, microsecond=0,
591 return datetime.datetime.utcnow().replace(second=0, microsecond=0,
579 minute=59, hour=23)
592 minute=59, hour=23)
580
593
581
594
582 class PermissiveDate(colander.DateTime):
595 class PermissiveDate(colander.DateTime):
583 """ Returns null for incorrect date format - also removes tz info"""
596 """ Returns null for incorrect date format - also removes tz info"""
584
597
585 def deserialize(self, node, cstruct):
598 def deserialize(self, node, cstruct):
586 if not cstruct:
599 if not cstruct:
587 return null
600 return null
588
601
589 try:
602 try:
590 result = colander.iso8601.parse_date(
603 result = colander.iso8601.parse_date(
591 cstruct, default_timezone=self.default_tzinfo)
604 cstruct, default_timezone=self.default_tzinfo)
592 except colander.iso8601.ParseError:
605 except colander.iso8601.ParseError:
593 return null
606 return null
594 return result.replace(tzinfo=None)
607 return result.replace(tzinfo=None)
595
608
596
609
597 class LogSearchSchema(colander.MappingSchema):
610 class LogSearchSchema(colander.MappingSchema):
598 def schema_type(self, **kw):
611 def schema_type(self, **kw):
599 return colander.Mapping(unknown='preserve')
612 return colander.Mapping(unknown='preserve')
600
613
601 resource = colander.SchemaNode(StringToAppList(),
614 resource = colander.SchemaNode(StringToAppList(),
602 validator=possible_applications_validator,
615 validator=possible_applications_validator,
603 missing=possible_applications)
616 missing=possible_applications)
604
617
605 message = colander.SchemaNode(colander.Sequence(accept_scalar=True),
618 message = colander.SchemaNode(colander.Sequence(accept_scalar=True),
606 colander.SchemaNode(colander.String()),
619 colander.SchemaNode(colander.String()),
607 missing=None)
620 missing=None)
608 level = colander.SchemaNode(colander.Sequence(accept_scalar=True),
621 level = colander.SchemaNode(colander.Sequence(accept_scalar=True),
609 colander.SchemaNode(colander.String()),
622 colander.SchemaNode(colander.String()),
610 preparer=lowercase_preparer,
623 preparer=lowercase_preparer,
611 missing=None)
624 missing=None)
612 namespace = colander.SchemaNode(colander.Sequence(accept_scalar=True),
625 namespace = colander.SchemaNode(colander.Sequence(accept_scalar=True),
613 colander.SchemaNode(colander.String()),
626 colander.SchemaNode(colander.String()),
614 preparer=lowercase_preparer,
627 preparer=lowercase_preparer,
615 missing=None)
628 missing=None)
616 request_id = colander.SchemaNode(colander.Sequence(accept_scalar=True),
629 request_id = colander.SchemaNode(colander.Sequence(accept_scalar=True),
617 colander.SchemaNode(colander.String()),
630 colander.SchemaNode(colander.String()),
618 preparer=lowercase_preparer,
631 preparer=lowercase_preparer,
619 missing=None)
632 missing=None)
620 start_date = colander.SchemaNode(PermissiveDate(),
633 start_date = colander.SchemaNode(PermissiveDate(),
621 missing=None)
634 missing=None)
622 end_date = colander.SchemaNode(PermissiveDate(),
635 end_date = colander.SchemaNode(PermissiveDate(),
623 missing=None)
636 missing=None)
624 page = colander.SchemaNode(colander.Integer(),
637 page = colander.SchemaNode(colander.Integer(),
625 validator=colander.Range(min=1),
638 validator=colander.Range(min=1),
626 missing=1)
639 missing=1)
627
640
628
641
629 class ReportSearchSchema(colander.MappingSchema):
642 class ReportSearchSchema(colander.MappingSchema):
630 def schema_type(self, **kw):
643 def schema_type(self, **kw):
631 return colander.Mapping(unknown='preserve')
644 return colander.Mapping(unknown='preserve')
632
645
633 resource = colander.SchemaNode(StringToAppList(),
646 resource = colander.SchemaNode(StringToAppList(),
634 validator=possible_applications_validator,
647 validator=possible_applications_validator,
635 missing=possible_applications)
648 missing=possible_applications)
636 request_id = colander.SchemaNode(colander.Sequence(accept_scalar=True),
649 request_id = colander.SchemaNode(colander.Sequence(accept_scalar=True),
637 colander.SchemaNode(colander.String()),
650 colander.SchemaNode(colander.String()),
638 missing=None)
651 missing=None)
639 start_date = colander.SchemaNode(PermissiveDate(),
652 start_date = colander.SchemaNode(PermissiveDate(),
640 missing=None)
653 missing=None)
641 end_date = colander.SchemaNode(PermissiveDate(),
654 end_date = colander.SchemaNode(PermissiveDate(),
642 missing=None)
655 missing=None)
643 page = colander.SchemaNode(colander.Integer(),
656 page = colander.SchemaNode(colander.Integer(),
644 validator=colander.Range(min=1),
657 validator=colander.Range(min=1),
645 missing=1)
658 missing=1)
646
659
647 min_occurences = colander.SchemaNode(
660 min_occurences = colander.SchemaNode(
648 colander.Sequence(accept_scalar=True),
661 colander.Sequence(accept_scalar=True),
649 colander.SchemaNode(colander.Integer()),
662 colander.SchemaNode(colander.Integer()),
650 missing=None)
663 missing=None)
651
664
652 http_status = colander.SchemaNode(colander.Sequence(accept_scalar=True),
665 http_status = colander.SchemaNode(colander.Sequence(accept_scalar=True),
653 colander.SchemaNode(colander.Integer()),
666 colander.SchemaNode(colander.Integer()),
654 missing=None)
667 missing=None)
655 priority = colander.SchemaNode(colander.Sequence(accept_scalar=True),
668 priority = colander.SchemaNode(colander.Sequence(accept_scalar=True),
656 colander.SchemaNode(colander.Integer()),
669 colander.SchemaNode(colander.Integer()),
657 missing=None)
670 missing=None)
658 error = colander.SchemaNode(colander.Sequence(accept_scalar=True),
671 error = colander.SchemaNode(colander.Sequence(accept_scalar=True),
659 colander.SchemaNode(colander.String()),
672 colander.SchemaNode(colander.String()),
660 missing=None)
673 missing=None)
661 url_path = colander.SchemaNode(colander.Sequence(accept_scalar=True),
674 url_path = colander.SchemaNode(colander.Sequence(accept_scalar=True),
662 colander.SchemaNode(colander.String()),
675 colander.SchemaNode(colander.String()),
663 missing=None)
676 missing=None)
664 url_domain = colander.SchemaNode(colander.Sequence(accept_scalar=True),
677 url_domain = colander.SchemaNode(colander.Sequence(accept_scalar=True),
665 colander.SchemaNode(colander.String()),
678 colander.SchemaNode(colander.String()),
666 missing=None)
679 missing=None)
667 report_status = colander.SchemaNode(colander.Sequence(accept_scalar=True),
680 report_status = colander.SchemaNode(colander.Sequence(accept_scalar=True),
668 colander.SchemaNode(colander.String()),
681 colander.SchemaNode(colander.String()),
669 missing=None)
682 missing=None)
670 min_duration = colander.SchemaNode(colander.Sequence(accept_scalar=True),
683 min_duration = colander.SchemaNode(colander.Sequence(accept_scalar=True),
671 colander.SchemaNode(colander.Float()),
684 colander.SchemaNode(colander.Float()),
672 missing=None)
685 missing=None)
673 max_duration = colander.SchemaNode(colander.Sequence(accept_scalar=True),
686 max_duration = colander.SchemaNode(colander.Sequence(accept_scalar=True),
674 colander.SchemaNode(colander.Float()),
687 colander.SchemaNode(colander.Float()),
675 missing=None)
688 missing=None)
676
689
677
690
678 class TagSchema(colander.MappingSchema):
691 class TagSchema(colander.MappingSchema):
679 """
692 """
680 Used in log search
693 Used in log search
681 """
694 """
682 name = colander.SchemaNode(colander.String(),
695 name = colander.SchemaNode(colander.String(),
683 validator=colander.Length(1, 32))
696 validator=colander.Length(1, 32))
684 value = colander.SchemaNode(colander.Sequence(accept_scalar=True),
697 value = colander.SchemaNode(colander.Sequence(accept_scalar=True),
685 colander.SchemaNode(colander.String(),
698 colander.SchemaNode(colander.String(),
686 validator=colander.Length(
699 validator=colander.Length(
687 1, 128)),
700 1, 128)),
688 missing=None)
701 missing=None)
689 op = colander.SchemaNode(colander.String(),
702 op = colander.SchemaNode(colander.String(),
690 validator=colander.Length(1, 128),
703 validator=colander.Length(1, 128),
691 missing=None)
704 missing=None)
692
705
693
706
694 class TagListSchema(colander.SequenceSchema):
707 class TagListSchema(colander.SequenceSchema):
695 tag = TagSchema()
708 tag = TagSchema()
696
709
697
710
698 class RuleFieldType(object):
711 class RuleFieldType(object):
699 """ Validator which succeeds if the value passed to it is one of
712 """ Validator which succeeds if the value passed to it is one of
700 a fixed set of values """
713 a fixed set of values """
701
714
702 def __init__(self, cast_to):
715 def __init__(self, cast_to):
703 self.cast_to = cast_to
716 self.cast_to = cast_to
704
717
705 def __call__(self, node, value):
718 def __call__(self, node, value):
706 try:
719 try:
707 if self.cast_to == 'int':
720 if self.cast_to == 'int':
708 int(value)
721 int(value)
709 elif self.cast_to == 'float':
722 elif self.cast_to == 'float':
710 float(value)
723 float(value)
711 elif self.cast_to == 'unicode':
724 elif self.cast_to == 'unicode':
712 str(value)
725 str(value)
713 except:
726 except:
714 raise colander.Invalid(node,
727 raise colander.Invalid(node,
715 "Can't cast {} to {}".format(
728 "Can't cast {} to {}".format(
716 value, self.cast_to))
729 value, self.cast_to))
717
730
718
731
719 def build_rule_schema(ruleset, check_matrix):
732 def build_rule_schema(ruleset, check_matrix):
720 """
733 """
721 Accepts ruleset and a map of fields/possible operations and builds
734 Accepts ruleset and a map of fields/possible operations and builds
722 validation class
735 validation class
723 """
736 """
724
737
725 schema = colander.SchemaNode(colander.Mapping())
738 schema = colander.SchemaNode(colander.Mapping())
726 schema.add(colander.SchemaNode(colander.String(), name='field'))
739 schema.add(colander.SchemaNode(colander.String(), name='field'))
727
740
728 if ruleset['field'] in ['__AND__', '__OR__', '__NOT__']:
741 if ruleset['field'] in ['__AND__', '__OR__', '__NOT__']:
729 subrules = colander.SchemaNode(colander.Tuple(), name='rules')
742 subrules = colander.SchemaNode(colander.Tuple(), name='rules')
730 for rule in ruleset['rules']:
743 for rule in ruleset['rules']:
731 subrules.add(build_rule_schema(rule, check_matrix))
744 subrules.add(build_rule_schema(rule, check_matrix))
732 schema.add(subrules)
745 schema.add(subrules)
733 else:
746 else:
734 op_choices = check_matrix[ruleset['field']]['ops']
747 op_choices = check_matrix[ruleset['field']]['ops']
735 cast_to = check_matrix[ruleset['field']]['type']
748 cast_to = check_matrix[ruleset['field']]['type']
736 schema.add(colander.SchemaNode(colander.String(),
749 schema.add(colander.SchemaNode(colander.String(),
737 validator=colander.OneOf(op_choices),
750 validator=colander.OneOf(op_choices),
738 name='op'))
751 name='op'))
739
752
740 schema.add(colander.SchemaNode(colander.String(),
753 schema.add(colander.SchemaNode(colander.String(),
741 name='value',
754 name='value',
742 validator=RuleFieldType(cast_to)))
755 validator=RuleFieldType(cast_to)))
743 return schema
756 return schema
744
757
745
758
746 class ConfigTypeSchema(colander.MappingSchema):
759 class ConfigTypeSchema(colander.MappingSchema):
747 type = colander.SchemaNode(colander.String(), missing=None)
760 type = colander.SchemaNode(colander.String(), missing=None)
748 config = colander.SchemaNode(UnknownType(), missing=None)
761 config = colander.SchemaNode(UnknownType(), missing=None)
749
762
750
763
751 class MappingListSchema(colander.SequenceSchema):
764 class MappingListSchema(colander.SequenceSchema):
752 config = colander.SchemaNode(UnknownType())
765 config = colander.SchemaNode(UnknownType())
@@ -1,427 +1,438 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2
2
3 # Copyright (C) 2010-2016 RhodeCode GmbH
3 # Copyright (C) 2010-2016 RhodeCode GmbH
4 #
4 #
5 # This program is free software: you can redistribute it and/or modify
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU Affero General Public License, version 3
6 # it under the terms of the GNU Affero General Public License, version 3
7 # (only), as published by the Free Software Foundation.
7 # (only), as published by the Free Software Foundation.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU Affero General Public License
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 #
16 #
17 # This program is dual-licensed. If you wish to learn more about the
17 # This program is dual-licensed. If you wish to learn more about the
18 # AppEnlight Enterprise Edition, including its added features, Support
18 # AppEnlight Enterprise Edition, including its added features, Support
19 # services, and proprietary license terms, please see
19 # services, and proprietary license terms, please see
20 # https://rhodecode.com/licenses/
20 # https://rhodecode.com/licenses/
21
21
22 import base64
22 import base64
23 import io
23 import io
24 import datetime
24 import datetime
25 import json
25 import json
26 import logging
26 import logging
27 import urllib.request, urllib.parse, urllib.error
27 import urllib.request, urllib.parse, urllib.error
28 import zlib
28 import zlib
29
29
30 from gzip import GzipFile
30 from gzip import GzipFile
31 from pyramid.view import view_config
31 from pyramid.view import view_config
32 from pyramid.httpexceptions import HTTPBadRequest
32 from pyramid.httpexceptions import HTTPBadRequest
33
33
34 import appenlight.celery.tasks as tasks
34 import appenlight.celery.tasks as tasks
35 from appenlight.lib.api import rate_limiting, check_cors
35 from appenlight.lib.api import rate_limiting, check_cors
36 from appenlight.lib.enums import ParsedSentryEventType
36 from appenlight.lib.enums import ParsedSentryEventType
37 from appenlight.lib.utils import parse_proto
37 from appenlight.lib.utils import parse_proto
38 from appenlight.lib.utils.airbrake import parse_airbrake_xml
38 from appenlight.lib.utils.airbrake import parse_airbrake_xml
39 from appenlight.lib.utils.date_utils import convert_date
39 from appenlight.lib.utils.date_utils import convert_date
40 from appenlight.lib.utils.sentry import parse_sentry_event
40 from appenlight.lib.utils.sentry import parse_sentry_event
41 from appenlight.lib.request import JSONException
41 from appenlight.lib.request import JSONException
42 from appenlight.validators import (LogListSchema,
42 from appenlight.validators import (LogListSchema,
43 MetricsListSchema,
43 MetricsListSchema,
44 GeneralMetricsListSchema,
44 GeneralMetricsListSchema,
45 GeneralMetricsPermanentListSchema,
45 GeneralMetricSchema,
46 GeneralMetricSchema,
47 GeneralMetricPermanentSchema,
46 LogListPermanentSchema,
48 LogListPermanentSchema,
47 ReportListSchema_0_5,
49 ReportListSchema_0_5,
48 LogSchema,
50 LogSchema,
49 LogSchemaPermanent,
51 LogSchemaPermanent,
50 ReportSchema_0_5)
52 ReportSchema_0_5)
51
53
52 log = logging.getLogger(__name__)
54 log = logging.getLogger(__name__)
53
55
54
56
55 @view_config(route_name='api_logs', renderer='string', permission='create',
57 @view_config(route_name='api_logs', renderer='string', permission='create',
56 require_csrf=False)
58 require_csrf=False)
57 @view_config(route_name='api_log', renderer='string', permission='create',
59 @view_config(route_name='api_log', renderer='string', permission='create',
58 require_csrf=False)
60 require_csrf=False)
59 def logs_create(request):
61 def logs_create(request):
60 """
62 """
61 Endpoint for log aggregation
63 Endpoint for log aggregation
62 """
64 """
63 application = request.context.resource
65 application = request.context.resource
64 if request.method.upper() == 'OPTIONS':
66 if request.method.upper() == 'OPTIONS':
65 return check_cors(request, application)
67 return check_cors(request, application)
66 else:
68 else:
67 check_cors(request, application, should_return=False)
69 check_cors(request, application, should_return=False)
68
70
69 params = dict(request.params.copy())
71 params = dict(request.params.copy())
70 proto_version = parse_proto(params.get('protocol_version', ''))
72 proto_version = parse_proto(params.get('protocol_version', ''))
71 payload = request.unsafe_json_body
73 payload = request.unsafe_json_body
72 sequence_accepted = request.matched_route.name == 'api_logs'
74 sequence_accepted = request.matched_route.name == 'api_logs'
73
75
74 if sequence_accepted:
76 if sequence_accepted:
75 if application.allow_permanent_storage:
77 if application.allow_permanent_storage:
76 schema = LogListPermanentSchema().bind(
78 schema = LogListPermanentSchema().bind(
77 utcnow=datetime.datetime.utcnow())
79 utcnow=datetime.datetime.utcnow())
78 else:
80 else:
79 schema = LogListSchema().bind(
81 schema = LogListSchema().bind(
80 utcnow=datetime.datetime.utcnow())
82 utcnow=datetime.datetime.utcnow())
81 else:
83 else:
82 if application.allow_permanent_storage:
84 if application.allow_permanent_storage:
83 schema = LogSchemaPermanent().bind(
85 schema = LogSchemaPermanent().bind(
84 utcnow=datetime.datetime.utcnow())
86 utcnow=datetime.datetime.utcnow())
85 else:
87 else:
86 schema = LogSchema().bind(
88 schema = LogSchema().bind(
87 utcnow=datetime.datetime.utcnow())
89 utcnow=datetime.datetime.utcnow())
88
90
89 deserialized_logs = schema.deserialize(payload)
91 deserialized_logs = schema.deserialize(payload)
90 if sequence_accepted is False:
92 if sequence_accepted is False:
91 deserialized_logs = [deserialized_logs]
93 deserialized_logs = [deserialized_logs]
92
94
93 rate_limiting(request, application, 'per_application_logs_rate_limit',
95 rate_limiting(request, application, 'per_application_logs_rate_limit',
94 len(deserialized_logs))
96 len(deserialized_logs))
95
97
96 # pprint.pprint(deserialized_logs)
98 # pprint.pprint(deserialized_logs)
97
99
98 # we need to split those out so we can process the pkey ones one by one
100 # we need to split those out so we can process the pkey ones one by one
99 non_pkey_logs = [log_dict for log_dict in deserialized_logs
101 non_pkey_logs = [log_dict for log_dict in deserialized_logs
100 if not log_dict['primary_key']]
102 if not log_dict['primary_key']]
101 pkey_dict = {}
103 pkey_dict = {}
102 # try to process the logs as best as we can and group together to reduce
104 # try to process the logs as best as we can and group together to reduce
103 # the amount of
105 # the amount of
104 for log_dict in deserialized_logs:
106 for log_dict in deserialized_logs:
105 if log_dict['primary_key']:
107 if log_dict['primary_key']:
106 key = (log_dict['primary_key'], log_dict['namespace'],)
108 key = (log_dict['primary_key'], log_dict['namespace'],)
107 if not key in pkey_dict:
109 if not key in pkey_dict:
108 pkey_dict[key] = []
110 pkey_dict[key] = []
109 pkey_dict[key].append(log_dict)
111 pkey_dict[key].append(log_dict)
110
112
111 if non_pkey_logs:
113 if non_pkey_logs:
112 log.debug('%s non-pkey logs received: %s' % (application,
114 log.debug('%s non-pkey logs received: %s' % (application,
113 len(non_pkey_logs)))
115 len(non_pkey_logs)))
114 tasks.add_logs.delay(application.resource_id, params, non_pkey_logs)
116 tasks.add_logs.delay(application.resource_id, params, non_pkey_logs)
115 if pkey_dict:
117 if pkey_dict:
116 logs_to_insert = []
118 logs_to_insert = []
117 for primary_key_tuple, payload in pkey_dict.items():
119 for primary_key_tuple, payload in pkey_dict.items():
118 sorted_logs = sorted(payload, key=lambda x: x['date'])
120 sorted_logs = sorted(payload, key=lambda x: x['date'])
119 logs_to_insert.append(sorted_logs[-1])
121 logs_to_insert.append(sorted_logs[-1])
120 log.debug('%s pkey logs received: %s' % (application,
122 log.debug('%s pkey logs received: %s' % (application,
121 len(logs_to_insert)))
123 len(logs_to_insert)))
122 tasks.add_logs.delay(application.resource_id, params, logs_to_insert)
124 tasks.add_logs.delay(application.resource_id, params, logs_to_insert)
123
125
124 log.info('LOG call %s %s client:%s' % (
126 log.info('LOG call %s %s client:%s' % (
125 application, proto_version, request.headers.get('user_agent')))
127 application, proto_version, request.headers.get('user_agent')))
126 return 'OK: Logs accepted'
128 return 'OK: Logs accepted'
127
129
128
130
129 @view_config(route_name='api_request_stats', renderer='string',
131 @view_config(route_name='api_request_stats', renderer='string',
130 permission='create', require_csrf=False)
132 permission='create', require_csrf=False)
131 @view_config(route_name='api_metrics', renderer='string',
133 @view_config(route_name='api_metrics', renderer='string',
132 permission='create', require_csrf=False)
134 permission='create', require_csrf=False)
133 def request_metrics_create(request):
135 def request_metrics_create(request):
134 """
136 """
135 Endpoint for performance metrics, aggregates view performance stats
137 Endpoint for performance metrics, aggregates view performance stats
136 and converts them to general metric row
138 and converts them to general metric row
137 """
139 """
138 application = request.context.resource
140 application = request.context.resource
139 if request.method.upper() == 'OPTIONS':
141 if request.method.upper() == 'OPTIONS':
140 return check_cors(request, application)
142 return check_cors(request, application)
141 else:
143 else:
142 check_cors(request, application, should_return=False)
144 check_cors(request, application, should_return=False)
143
145
144 params = dict(request.params.copy())
146 params = dict(request.params.copy())
145 proto_version = parse_proto(params.get('protocol_version', ''))
147 proto_version = parse_proto(params.get('protocol_version', ''))
146
148
147 payload = request.unsafe_json_body
149 payload = request.unsafe_json_body
148 schema = MetricsListSchema()
150 schema = MetricsListSchema()
149 dataset = schema.deserialize(payload)
151 dataset = schema.deserialize(payload)
150
152
151 rate_limiting(request, application, 'per_application_metrics_rate_limit',
153 rate_limiting(request, application, 'per_application_metrics_rate_limit',
152 len(dataset))
154 len(dataset))
153
155
154 # looping report data
156 # looping report data
155 metrics = {}
157 metrics = {}
156 for metric in dataset:
158 for metric in dataset:
157 server_name = metric.get('server', '').lower() or 'unknown'
159 server_name = metric.get('server', '').lower() or 'unknown'
158 start_interval = convert_date(metric['timestamp'])
160 start_interval = convert_date(metric['timestamp'])
159 start_interval = start_interval.replace(second=0, microsecond=0)
161 start_interval = start_interval.replace(second=0, microsecond=0)
160
162
161 for view_name, view_metrics in metric['metrics']:
163 for view_name, view_metrics in metric['metrics']:
162 key = '%s%s%s' % (metric['server'], start_interval, view_name)
164 key = '%s%s%s' % (metric['server'], start_interval, view_name)
163 if start_interval not in metrics:
165 if start_interval not in metrics:
164 metrics[key] = {"requests": 0, "main": 0, "sql": 0,
166 metrics[key] = {"requests": 0, "main": 0, "sql": 0,
165 "nosql": 0, "remote": 0, "tmpl": 0,
167 "nosql": 0, "remote": 0, "tmpl": 0,
166 "custom": 0, 'sql_calls': 0,
168 "custom": 0, 'sql_calls': 0,
167 'nosql_calls': 0,
169 'nosql_calls': 0,
168 'remote_calls': 0, 'tmpl_calls': 0,
170 'remote_calls': 0, 'tmpl_calls': 0,
169 'custom_calls': 0,
171 'custom_calls': 0,
170 "start_interval": start_interval,
172 "start_interval": start_interval,
171 "server_name": server_name,
173 "server_name": server_name,
172 "view_name": view_name
174 "view_name": view_name
173 }
175 }
174 metrics[key]["requests"] += int(view_metrics['requests'])
176 metrics[key]["requests"] += int(view_metrics['requests'])
175 metrics[key]["main"] += round(view_metrics['main'], 5)
177 metrics[key]["main"] += round(view_metrics['main'], 5)
176 metrics[key]["sql"] += round(view_metrics['sql'], 5)
178 metrics[key]["sql"] += round(view_metrics['sql'], 5)
177 metrics[key]["nosql"] += round(view_metrics['nosql'], 5)
179 metrics[key]["nosql"] += round(view_metrics['nosql'], 5)
178 metrics[key]["remote"] += round(view_metrics['remote'], 5)
180 metrics[key]["remote"] += round(view_metrics['remote'], 5)
179 metrics[key]["tmpl"] += round(view_metrics['tmpl'], 5)
181 metrics[key]["tmpl"] += round(view_metrics['tmpl'], 5)
180 metrics[key]["custom"] += round(view_metrics.get('custom', 0.0),
182 metrics[key]["custom"] += round(view_metrics.get('custom', 0.0),
181 5)
183 5)
182 metrics[key]["sql_calls"] += int(
184 metrics[key]["sql_calls"] += int(
183 view_metrics.get('sql_calls', 0))
185 view_metrics.get('sql_calls', 0))
184 metrics[key]["nosql_calls"] += int(
186 metrics[key]["nosql_calls"] += int(
185 view_metrics.get('nosql_calls', 0))
187 view_metrics.get('nosql_calls', 0))
186 metrics[key]["remote_calls"] += int(
188 metrics[key]["remote_calls"] += int(
187 view_metrics.get('remote_calls', 0))
189 view_metrics.get('remote_calls', 0))
188 metrics[key]["tmpl_calls"] += int(
190 metrics[key]["tmpl_calls"] += int(
189 view_metrics.get('tmpl_calls', 0))
191 view_metrics.get('tmpl_calls', 0))
190 metrics[key]["custom_calls"] += int(
192 metrics[key]["custom_calls"] += int(
191 view_metrics.get('custom_calls', 0))
193 view_metrics.get('custom_calls', 0))
192
194
193 if not metrics[key]["requests"]:
195 if not metrics[key]["requests"]:
194 # fix this here because validator can't
196 # fix this here because validator can't
195 metrics[key]["requests"] = 1
197 metrics[key]["requests"] = 1
196 # metrics dict is being built to minimize
198 # metrics dict is being built to minimize
197 # the amount of queries used
199 # the amount of queries used
198 # in case we get multiple rows from same minute
200 # in case we get multiple rows from same minute
199
201
200 normalized_metrics = []
202 normalized_metrics = []
201 for metric in metrics.values():
203 for metric in metrics.values():
202 new_metric = {
204 new_metric = {
203 'namespace': 'appenlight.request_metric',
205 'namespace': 'appenlight.request_metric',
204 'timestamp': metric.pop('start_interval'),
206 'timestamp': metric.pop('start_interval'),
205 'server_name': metric['server_name'],
207 'server_name': metric['server_name'],
206 'tags': list(metric.items())
208 'tags': list(metric.items())
207 }
209 }
208 normalized_metrics.append(new_metric)
210 normalized_metrics.append(new_metric)
209
211
210 tasks.add_metrics.delay(application.resource_id, params,
212 tasks.add_metrics.delay(application.resource_id, params,
211 normalized_metrics, proto_version)
213 normalized_metrics, proto_version)
212
214
213 log.info('REQUEST METRICS call {} {} client:{}'.format(
215 log.info('REQUEST METRICS call {} {} client:{}'.format(
214 application.resource_name, proto_version,
216 application.resource_name, proto_version,
215 request.headers.get('user_agent')))
217 request.headers.get('user_agent')))
216 return 'OK: request metrics accepted'
218 return 'OK: request metrics accepted'
217
219
218
220
219 @view_config(route_name='api_general_metrics', renderer='string',
221 @view_config(route_name='api_general_metrics', renderer='string',
220 permission='create', require_csrf=False)
222 permission='create', require_csrf=False)
221 @view_config(route_name='api_general_metric', renderer='string',
223 @view_config(route_name='api_general_metric', renderer='string',
222 permission='create', require_csrf=False)
224 permission='create', require_csrf=False)
223 def general_metrics_create(request):
225 def general_metrics_create(request):
224 """
226 """
225 Endpoint for general metrics aggregation
227 Endpoint for general metrics aggregation
226 """
228 """
227 application = request.context.resource
229 application = request.context.resource
228 if request.method.upper() == 'OPTIONS':
230 if request.method.upper() == 'OPTIONS':
229 return check_cors(request, application)
231 return check_cors(request, application)
230 else:
232 else:
231 check_cors(request, application, should_return=False)
233 check_cors(request, application, should_return=False)
232
234
233 params = dict(request.params.copy())
235 params = dict(request.params.copy())
234 proto_version = parse_proto(params.get('protocol_version', ''))
236 proto_version = parse_proto(params.get('protocol_version', ''))
235 payload = request.unsafe_json_body
237 payload = request.unsafe_json_body
236 sequence_accepted = request.matched_route.name == 'api_general_metrics'
238 sequence_accepted = request.matched_route.name == 'api_general_metrics'
237 if sequence_accepted:
239 if sequence_accepted:
238 schema = GeneralMetricsListSchema().bind(
240 if application.allow_permanent_storage:
239 utcnow=datetime.datetime.utcnow())
241 schema = GeneralMetricsPermanentListSchema().bind(
242 utcnow=datetime.datetime.utcnow())
243 else:
244 schema = GeneralMetricsListSchema().bind(
245 utcnow=datetime.datetime.utcnow())
240 else:
246 else:
241 schema = GeneralMetricSchema().bind(utcnow=datetime.datetime.utcnow())
247 if application.allow_permanent_storage:
248 schema = GeneralMetricPermanentSchema().bind(
249 utcnow=datetime.datetime.utcnow())
250 else:
251 schema = GeneralMetricSchema().bind(
252 utcnow=datetime.datetime.utcnow())
242
253
243 deserialized_metrics = schema.deserialize(payload)
254 deserialized_metrics = schema.deserialize(payload)
244 if sequence_accepted is False:
255 if sequence_accepted is False:
245 deserialized_metrics = [deserialized_metrics]
256 deserialized_metrics = [deserialized_metrics]
246
257
247 rate_limiting(request, application, 'per_application_metrics_rate_limit',
258 rate_limiting(request, application, 'per_application_metrics_rate_limit',
248 len(deserialized_metrics))
259 len(deserialized_metrics))
249
260
250 tasks.add_metrics.delay(application.resource_id, params,
261 tasks.add_metrics.delay(application.resource_id, params,
251 deserialized_metrics, proto_version)
262 deserialized_metrics, proto_version)
252
263
253 log.info('METRICS call {} {} client:{}'.format(
264 log.info('METRICS call {} {} client:{}'.format(
254 application.resource_name, proto_version,
265 application.resource_name, proto_version,
255 request.headers.get('user_agent')))
266 request.headers.get('user_agent')))
256 return 'OK: Metrics accepted'
267 return 'OK: Metrics accepted'
257
268
258
269
259 @view_config(route_name='api_reports', renderer='string', permission='create',
270 @view_config(route_name='api_reports', renderer='string', permission='create',
260 require_csrf=False)
271 require_csrf=False)
261 @view_config(route_name='api_slow_reports', renderer='string',
272 @view_config(route_name='api_slow_reports', renderer='string',
262 permission='create', require_csrf=False)
273 permission='create', require_csrf=False)
263 @view_config(route_name='api_report', renderer='string', permission='create',
274 @view_config(route_name='api_report', renderer='string', permission='create',
264 require_csrf=False)
275 require_csrf=False)
265 def reports_create(request):
276 def reports_create(request):
266 """
277 """
267 Endpoint for exception and slowness reports
278 Endpoint for exception and slowness reports
268 """
279 """
269 # route_url('reports')
280 # route_url('reports')
270 application = request.context.resource
281 application = request.context.resource
271 if request.method.upper() == 'OPTIONS':
282 if request.method.upper() == 'OPTIONS':
272 return check_cors(request, application)
283 return check_cors(request, application)
273 else:
284 else:
274 check_cors(request, application, should_return=False)
285 check_cors(request, application, should_return=False)
275 params = dict(request.params.copy())
286 params = dict(request.params.copy())
276 proto_version = parse_proto(params.get('protocol_version', ''))
287 proto_version = parse_proto(params.get('protocol_version', ''))
277 payload = request.unsafe_json_body
288 payload = request.unsafe_json_body
278 sequence_accepted = request.matched_route.name == 'api_reports'
289 sequence_accepted = request.matched_route.name == 'api_reports'
279
290
280 if sequence_accepted:
291 if sequence_accepted:
281 schema = ReportListSchema_0_5().bind(
292 schema = ReportListSchema_0_5().bind(
282 utcnow=datetime.datetime.utcnow())
293 utcnow=datetime.datetime.utcnow())
283 else:
294 else:
284 schema = ReportSchema_0_5().bind(
295 schema = ReportSchema_0_5().bind(
285 utcnow=datetime.datetime.utcnow())
296 utcnow=datetime.datetime.utcnow())
286
297
287 deserialized_reports = schema.deserialize(payload)
298 deserialized_reports = schema.deserialize(payload)
288 if sequence_accepted is False:
299 if sequence_accepted is False:
289 deserialized_reports = [deserialized_reports]
300 deserialized_reports = [deserialized_reports]
290 if deserialized_reports:
301 if deserialized_reports:
291 rate_limiting(request, application,
302 rate_limiting(request, application,
292 'per_application_reports_rate_limit',
303 'per_application_reports_rate_limit',
293 len(deserialized_reports))
304 len(deserialized_reports))
294
305
295 # pprint.pprint(deserialized_reports)
306 # pprint.pprint(deserialized_reports)
296 tasks.add_reports.delay(application.resource_id, params,
307 tasks.add_reports.delay(application.resource_id, params,
297 deserialized_reports)
308 deserialized_reports)
298 log.info('REPORT call %s, %s client:%s' % (
309 log.info('REPORT call %s, %s client:%s' % (
299 application,
310 application,
300 proto_version,
311 proto_version,
301 request.headers.get('user_agent'))
312 request.headers.get('user_agent'))
302 )
313 )
303 return 'OK: Reports accepted'
314 return 'OK: Reports accepted'
304
315
305
316
306 @view_config(route_name='api_airbrake', renderer='string', permission='create',
317 @view_config(route_name='api_airbrake', renderer='string', permission='create',
307 require_csrf=False)
318 require_csrf=False)
308 def airbrake_xml_compat(request):
319 def airbrake_xml_compat(request):
309 """
320 """
310 Airbrake compatible endpoint for XML reports
321 Airbrake compatible endpoint for XML reports
311 """
322 """
312 application = request.context.resource
323 application = request.context.resource
313 if request.method.upper() == 'OPTIONS':
324 if request.method.upper() == 'OPTIONS':
314 return check_cors(request, application)
325 return check_cors(request, application)
315 else:
326 else:
316 check_cors(request, application, should_return=False)
327 check_cors(request, application, should_return=False)
317
328
318 params = request.params.copy()
329 params = request.params.copy()
319
330
320 error_dict = parse_airbrake_xml(request)
331 error_dict = parse_airbrake_xml(request)
321 schema = ReportListSchema_0_5().bind(utcnow=datetime.datetime.utcnow())
332 schema = ReportListSchema_0_5().bind(utcnow=datetime.datetime.utcnow())
322 deserialized_reports = schema.deserialize([error_dict])
333 deserialized_reports = schema.deserialize([error_dict])
323 rate_limiting(request, application, 'per_application_reports_rate_limit',
334 rate_limiting(request, application, 'per_application_reports_rate_limit',
324 len(deserialized_reports))
335 len(deserialized_reports))
325
336
326 tasks.add_reports.delay(application.resource_id, params,
337 tasks.add_reports.delay(application.resource_id, params,
327 deserialized_reports)
338 deserialized_reports)
328 log.info('%s AIRBRAKE call for application %s, api_ver:%s client:%s' % (
339 log.info('%s AIRBRAKE call for application %s, api_ver:%s client:%s' % (
329 500, application.resource_name,
340 500, application.resource_name,
330 request.params.get('protocol_version', 'unknown'),
341 request.params.get('protocol_version', 'unknown'),
331 request.headers.get('user_agent'))
342 request.headers.get('user_agent'))
332 )
343 )
333 return '<notice><id>no-id</id><url>%s</url></notice>' % \
344 return '<notice><id>no-id</id><url>%s</url></notice>' % \
334 request.registry.settings['mailing.app_url']
345 request.registry.settings['mailing.app_url']
335
346
336
347
337 def decompress_gzip(data):
348 def decompress_gzip(data):
338 try:
349 try:
339 fp = io.StringIO(data)
350 fp = io.StringIO(data)
340 with GzipFile(fileobj=fp) as f:
351 with GzipFile(fileobj=fp) as f:
341 return f.read()
352 return f.read()
342 except Exception as exc:
353 except Exception as exc:
343 raise
354 raise
344 log.error(exc)
355 log.error(exc)
345 raise HTTPBadRequest()
356 raise HTTPBadRequest()
346
357
347
358
348 def decompress_zlib(data):
359 def decompress_zlib(data):
349 try:
360 try:
350 return zlib.decompress(data)
361 return zlib.decompress(data)
351 except Exception as exc:
362 except Exception as exc:
352 raise
363 raise
353 log.error(exc)
364 log.error(exc)
354 raise HTTPBadRequest()
365 raise HTTPBadRequest()
355
366
356
367
357 def decode_b64(data):
368 def decode_b64(data):
358 try:
369 try:
359 return base64.b64decode(data)
370 return base64.b64decode(data)
360 except Exception as exc:
371 except Exception as exc:
361 raise
372 raise
362 log.error(exc)
373 log.error(exc)
363 raise HTTPBadRequest()
374 raise HTTPBadRequest()
364
375
365
376
366 @view_config(route_name='api_sentry', renderer='string', permission='create',
377 @view_config(route_name='api_sentry', renderer='string', permission='create',
367 require_csrf=False)
378 require_csrf=False)
368 @view_config(route_name='api_sentry_slash', renderer='string',
379 @view_config(route_name='api_sentry_slash', renderer='string',
369 permission='create', require_csrf=False)
380 permission='create', require_csrf=False)
370 def sentry_compat(request):
381 def sentry_compat(request):
371 """
382 """
372 Sentry compatible endpoint
383 Sentry compatible endpoint
373 """
384 """
374 application = request.context.resource
385 application = request.context.resource
375 if request.method.upper() == 'OPTIONS':
386 if request.method.upper() == 'OPTIONS':
376 return check_cors(request, application)
387 return check_cors(request, application)
377 else:
388 else:
378 check_cors(request, application, should_return=False)
389 check_cors(request, application, should_return=False)
379
390
380 # handle various report encoding
391 # handle various report encoding
381 content_encoding = request.headers.get('Content-Encoding')
392 content_encoding = request.headers.get('Content-Encoding')
382 content_type = request.headers.get('Content-Type')
393 content_type = request.headers.get('Content-Type')
383 if content_encoding == 'gzip':
394 if content_encoding == 'gzip':
384 body = decompress_gzip(request.body)
395 body = decompress_gzip(request.body)
385 elif content_encoding == 'deflate':
396 elif content_encoding == 'deflate':
386 body = decompress_zlib(request.body)
397 body = decompress_zlib(request.body)
387 else:
398 else:
388 body = request.body
399 body = request.body
389 # attempt to fix string before decoding for stupid clients
400 # attempt to fix string before decoding for stupid clients
390 if content_type == 'application/x-www-form-urlencoded':
401 if content_type == 'application/x-www-form-urlencoded':
391 body = urllib.parse.unquote(body.decode('utf8'))
402 body = urllib.parse.unquote(body.decode('utf8'))
392 check_char = '{' if isinstance(body, str) else b'{'
403 check_char = '{' if isinstance(body, str) else b'{'
393 if not body.startswith(check_char):
404 if not body.startswith(check_char):
394 try:
405 try:
395 body = decode_b64(body)
406 body = decode_b64(body)
396 body = decompress_zlib(body)
407 body = decompress_zlib(body)
397 except Exception as exc:
408 except Exception as exc:
398 log.info(exc)
409 log.info(exc)
399
410
400 try:
411 try:
401 json_body = json.loads(body.decode('utf8'))
412 json_body = json.loads(body.decode('utf8'))
402 except ValueError:
413 except ValueError:
403 raise JSONException("Incorrect JSON")
414 raise JSONException("Incorrect JSON")
404
415
405 event, event_type = parse_sentry_event(json_body)
416 event, event_type = parse_sentry_event(json_body)
406
417
407 if event_type == ParsedSentryEventType.LOG:
418 if event_type == ParsedSentryEventType.LOG:
408 if application.allow_permanent_storage:
419 if application.allow_permanent_storage:
409 schema = LogSchemaPermanent().bind(
420 schema = LogSchemaPermanent().bind(
410 utcnow=datetime.datetime.utcnow())
421 utcnow=datetime.datetime.utcnow())
411 else:
422 else:
412 schema = LogSchema().bind(
423 schema = LogSchema().bind(
413 utcnow=datetime.datetime.utcnow())
424 utcnow=datetime.datetime.utcnow())
414 deserialized_logs = schema.deserialize(event)
425 deserialized_logs = schema.deserialize(event)
415 non_pkey_logs = [deserialized_logs]
426 non_pkey_logs = [deserialized_logs]
416 log.debug('%s non-pkey logs received: %s' % (application,
427 log.debug('%s non-pkey logs received: %s' % (application,
417 len(non_pkey_logs)))
428 len(non_pkey_logs)))
418 tasks.add_logs.delay(application.resource_id, {}, non_pkey_logs)
429 tasks.add_logs.delay(application.resource_id, {}, non_pkey_logs)
419 if event_type == ParsedSentryEventType.ERROR_REPORT:
430 if event_type == ParsedSentryEventType.ERROR_REPORT:
420 schema = ReportSchema_0_5().bind(utcnow=datetime.datetime.utcnow())
431 schema = ReportSchema_0_5().bind(utcnow=datetime.datetime.utcnow())
421 deserialized_reports = [schema.deserialize(event)]
432 deserialized_reports = [schema.deserialize(event)]
422 rate_limiting(request, application,
433 rate_limiting(request, application,
423 'per_application_reports_rate_limit',
434 'per_application_reports_rate_limit',
424 len(deserialized_reports))
435 len(deserialized_reports))
425 tasks.add_reports.delay(application.resource_id, {},
436 tasks.add_reports.delay(application.resource_id, {},
426 deserialized_reports)
437 deserialized_reports)
427 return 'OK: Events accepted'
438 return 'OK: Events accepted'
General Comments 0
You need to be logged in to leave comments. Login now