From 28583a5e4c779e0674d4b04f7b0367c0bdaf2716 2016-08-22 12:51:53 From: Marcin Lulek Date: 2016-08-22 12:51:53 Subject: [PATCH] api: support permanent logs in sentry api --- diff --git a/backend/src/appenlight/tests/test_unit.py b/backend/src/appenlight/tests/test_unit.py index 50c9aaa..ace3685 100644 --- a/backend/src/appenlight/tests/test_unit.py +++ b/backend/src/appenlight/tests/test_unit.py @@ -53,116 +53,6 @@ class TestMigration(object): assert 1 == 1 -class TestAPIReports_0_4_Validation(object): - @pytest.mark.parametrize('dummy_json', ['', {}, [], None]) - def test_no_payload(self, dummy_json): - import colander - from appenlight.validators import ReportListSchema_0_4 - utcnow = datetime.utcnow() - schema = ReportListSchema_0_4().bind(utcnow=utcnow) - with pytest.raises(colander.Invalid): - schema.deserialize(dummy_json) - - def test_minimal_payload(self, report_04_schema): - dummy_json = [{}] - import colander - from appenlight.validators import ReportListSchema_0_4 - utcnow = datetime.utcnow() - schema = ReportListSchema_0_4().bind(utcnow=utcnow) - with pytest.raises(colander.Invalid): - schema.deserialize(dummy_json) - - def test_minimal_payload(self): - from appenlight.validators import ReportListSchema_0_4 - dummy_json = [{'report_details': [{}]}] - utcnow = datetime.utcnow() - schema = ReportListSchema_0_4().bind(utcnow=utcnow) - deserialized = schema.deserialize(dummy_json) - - expected_deserialization = [ - {'error_type': '', - 'language': 'unknown', - 'report_details': [ - {'username': '', - 'traceback': None, - 'extra': None, - 'frameinfo': None, - 'url': '', - 'ip': None, - 'start_time': utcnow, - 'group_string': None, - 'request': {}, - 'request_stats': None, - 'end_time': None, - 'request_id': '', - 'message': '', - 'slow_calls': [], - 'user_agent': ''}], - 'server': 'unknown', - 'occurences': 1, - 'priority': 5, - 'view_name': '', - 'client': 'unknown', - 'http_status': 200, - 'error': '', - 'tags': None} - ] - assert deserialized == expected_deserialization - - def test_full_payload(self): - import appenlight.tests.payload_examples as payload_examples - from appenlight.validators import ReportListSchema_0_4 - utcnow = datetime.utcnow() - schema = ReportListSchema_0_4().bind(utcnow=utcnow) - PYTHON_PAYLOAD = copy.deepcopy(payload_examples.PYTHON_PAYLOAD_0_4) - utcnow = datetime.utcnow() - PYTHON_PAYLOAD["tags"] = [("foo", 1), ("action", "test"), ("baz", 1.1), - ("date", - utcnow.strftime('%Y-%m-%dT%H:%M:%S.0'))] - dummy_json = [PYTHON_PAYLOAD] - - deserialized = schema.deserialize(dummy_json) - assert deserialized[0]['error'] == PYTHON_PAYLOAD['error'] - assert deserialized[0]['language'] == PYTHON_PAYLOAD['language'] - assert deserialized[0]['server'] == PYTHON_PAYLOAD['server'] - assert deserialized[0]['priority'] == PYTHON_PAYLOAD['priority'] - assert deserialized[0]['view_name'] == PYTHON_PAYLOAD['view_name'] - assert deserialized[0]['client'] == PYTHON_PAYLOAD['client'] - assert deserialized[0]['http_status'] == PYTHON_PAYLOAD['http_status'] - assert deserialized[0]['error'] == PYTHON_PAYLOAD['error'] - assert deserialized[0]['occurences'] == PYTHON_PAYLOAD['occurences'] - first_detail = deserialized[0]['report_details'][0] - payload_detail = PYTHON_PAYLOAD['report_details'][0] - assert first_detail['username'] == payload_detail['username'] - assert first_detail['traceback'] == payload_detail['traceback'] - assert first_detail['url'] == payload_detail['url'] - assert first_detail['ip'] == payload_detail['ip'] - assert first_detail['start_time'].strftime('%Y-%m-%dT%H:%M:%S.0') == \ - payload_detail['start_time'] - assert first_detail['ip'] == payload_detail['ip'] - assert first_detail['group_string'] is None - assert first_detail['request_stats'] == payload_detail['request_stats'] - assert first_detail['end_time'].strftime('%Y-%m-%dT%H:%M:%S.0') == \ - payload_detail['end_time'] - assert first_detail['request_id'] == payload_detail['request_id'] - assert first_detail['message'] == payload_detail['message'] - assert first_detail['user_agent'] == payload_detail['user_agent'] - slow_call = first_detail['slow_calls'][0] - expected_slow_call = payload_detail['slow_calls'][0] - assert slow_call['start'].strftime('%Y-%m-%dT%H:%M:%S.0') == \ - expected_slow_call['start'] - assert slow_call['end'].strftime('%Y-%m-%dT%H:%M:%S.0') == \ - expected_slow_call['end'] - assert slow_call['statement'] == expected_slow_call['statement'] - assert slow_call['parameters'] == expected_slow_call['parameters'] - assert slow_call['type'] == expected_slow_call['type'] - assert slow_call['subtype'] == expected_slow_call['subtype'] - assert slow_call['location'] == '' - assert deserialized[0]['tags'] == [ - ('foo', 1), ('action', 'test'), - ('baz', 1.1), ('date', utcnow.strftime('%Y-%m-%dT%H:%M:%S.0'))] - - class TestSentryProto_7(object): def test_log_payload(self): import appenlight.tests.payload_examples as payload_examples @@ -461,11 +351,11 @@ class TestAPIGeneralMetricsValidation(object): general_metrics_schema.deserialize(dummy_json) def test_minimal_payload(self, general_metrics_schema): - dummy_json = [{}] + dummy_json = [{'tags': [['counter_a', 15.5], ['counter_b', 63]]}] deserialized = general_metrics_schema.deserialize(dummy_json)[0] expected = {'namespace': '', 'server_name': 'unknown', - 'tags': None, + 'tags': [('counter_a', 15.5), ('counter_b', 63)], 'timestamp': datetime.utcnow()} assert deserialized['namespace'] == expected['namespace'] assert deserialized['server_name'] == expected['server_name'] diff --git a/backend/src/appenlight/validators.py b/backend/src/appenlight/validators.py index 677f691..263fcdc 100644 --- a/backend/src/appenlight/validators.py +++ b/backend/src/appenlight/validators.py @@ -192,17 +192,20 @@ class SlowCallSchema(colander.MappingSchema): def limited_date(node, value): """ checks to make sure that the value is not older/newer than 2h """ - hours = 2 - min_time = datetime.datetime.utcnow() - datetime.timedelta(hours=72) - max_time = datetime.datetime.utcnow() + datetime.timedelta(hours=2) + past_hours = 72 + future_hours = 2 + min_time = datetime.datetime.utcnow() - datetime.timedelta( + hours=past_hours) + max_time = datetime.datetime.utcnow() + datetime.timedelta( + hours=future_hours) if min_time > value: - msg = '%r is older from current UTC time by ' + str(hours) + ' hours.' - msg += ' Ask administrator to enable permanent logging for ' \ + msg = '%r is older from current UTC time by ' + str(past_hours) + msg += ' hours. Ask administrator to enable permanent logging for ' \ 'your application to store logs with dates in past.' raise colander.Invalid(node, msg % value) if max_time < value: - msg = '%r is newer from current UTC time by ' + str(hours) + ' hours' - msg += ' Ask administrator to enable permanent logging for ' \ + msg = '%r is newer from current UTC time by ' + str(future_hours) + msg += ' hours. Ask administrator to enable permanent logging for ' \ 'your application to store logs with dates in future.' raise colander.Invalid(node, msg % value) @@ -311,20 +314,13 @@ class ReportDetailBaseSchema(colander.MappingSchema): extra = ExtraSchemaList() -class ReportDetailSchema_0_4(ReportDetailBaseSchema): - frameinfo = FrameInfoListSchema(missing=None) - - class ReportDetailSchema_0_5(ReportDetailBaseSchema): pass -class ReportDetailListSchema(colander.SequenceSchema): - """ - Validates format of list of reports - """ - report_detail = ReportDetailSchema_0_4() - validator = colander.Length(1) +class ReportDetailSchemaPermissiveDate_0_5(ReportDetailSchema_0_5): + start_time = colander.SchemaNode(NonTZDate(), missing=deferred_utcnow) + end_time = colander.SchemaNode(NonTZDate(), missing=None) class ReportSchemaBase(colander.MappingSchema): @@ -364,6 +360,11 @@ class ReportSchema_0_5(ReportSchemaBase, ReportDetailSchema_0_5): pass +class ReportSchemaPermissiveDate_0_5(ReportSchemaBase, + ReportDetailSchemaPermissiveDate_0_5): + pass + + class ReportListSchema_0_5(colander.SequenceSchema): """ Validates format of list of report groups @@ -372,6 +373,14 @@ class ReportListSchema_0_5(colander.SequenceSchema): validator = colander.Length(1) +class ReportListPermissiveDateSchema_0_5(colander.SequenceSchema): + """ + Validates format of list of report groups + """ + report = ReportSchemaPermissiveDate_0_5() + validator = colander.Length(1) + + class LogSchema(colander.MappingSchema): """ Validates format if individual log entry diff --git a/backend/src/appenlight/views/api.py b/backend/src/appenlight/views/api.py index b5213e4..cf5bd12 100644 --- a/backend/src/appenlight/views/api.py +++ b/backend/src/appenlight/views/api.py @@ -405,7 +405,12 @@ def sentry_compat(request): event, event_type = parse_sentry_event(json_body) if event_type == ParsedSentryEventType.LOG: - schema = LogSchema().bind(utcnow=datetime.datetime.utcnow()) + if application.allow_permanent_storage: + schema = LogSchemaPermanent().bind( + utcnow=datetime.datetime.utcnow()) + else: + schema = LogSchema().bind( + utcnow=datetime.datetime.utcnow()) deserialized_logs = schema.deserialize(event) non_pkey_logs = [deserialized_logs] log.debug('%s non-pkey logs received: %s' % (application,