From 87908169d8e0102e995a2a42813a0f318b153b9f 2016-06-03 12:14:37 From: Marcin Lulek Date: 2016-06-03 12:14:37 Subject: [PATCH] project: add missing lib directory --- diff --git a/.hgignore b/.hgignore index d8b23d4..8a7e7e6 100644 --- a/.hgignore +++ b/.hgignore @@ -26,7 +26,7 @@ dist/ downloads/ eggs/ .eggs/ -lib/ +$lib lib64/ parts/ sdist/ diff --git a/backend/src/appenlight/lib/__init__.py b/backend/src/appenlight/lib/__init__.py new file mode 100644 index 0000000..27c5848 --- /dev/null +++ b/backend/src/appenlight/lib/__init__.py @@ -0,0 +1,53 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2010-2016 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# App Enlight Enterprise Edition, including its added features, Support +# services, and proprietary license terms, please see +# https://rhodecode.com/licenses/ + +"""Miscellaneous support packages for {{project}}. +""" +import random +import string +import importlib + +from appenlight_client.exceptions import get_current_traceback + + +def generate_random_string(chars=10): + return ''.join(random.sample(string.ascii_letters * 2 + string.digits, + chars)) + + +def to_integer_safe(input): + try: + return int(input) + except (TypeError, ValueError,): + return None + +def print_traceback(log): + traceback = get_current_traceback(skip=1, show_hidden_frames=True, + ignore_system_exceptions=True) + exception_text = traceback.exception + log.error(exception_text) + log.error(traceback.plaintext) + del traceback + +def get_callable(import_string): + import_module, indexer_callable = import_string.split(':') + return getattr(importlib.import_module(import_module), + indexer_callable) diff --git a/backend/src/appenlight/lib/api.py b/backend/src/appenlight/lib/api.py new file mode 100644 index 0000000..5885215 --- /dev/null +++ b/backend/src/appenlight/lib/api.py @@ -0,0 +1,83 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2010-2016 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# App Enlight Enterprise Edition, including its added features, Support +# services, and proprietary license terms, please see +# https://rhodecode.com/licenses/ + +import datetime +import logging + +from pyramid.httpexceptions import HTTPForbidden, HTTPTooManyRequests + +from appenlight.models import Datastores +from appenlight.models.services.config import ConfigService +from appenlight.lib.redis_keys import REDIS_KEYS + +log = logging.getLogger(__name__) + + +def rate_limiting(request, resource, section, to_increment=1): + tsample = datetime.datetime.utcnow().replace(second=0, microsecond=0) + key = REDIS_KEYS['rate_limits'][section].format(tsample, + resource.resource_id) + current_count = Datastores.redis.incr(key, to_increment) + Datastores.redis.expire(key, 3600 * 24) + config = ConfigService.by_key_and_section(section, 'global') + limit = config.value if config else 1000 + if current_count > int(limit): + log.info('RATE LIMITING: {}: {}, {}'.format( + section, resource, current_count)) + abort_msg = 'Rate limits are in effect for this application' + raise HTTPTooManyRequests(abort_msg, + headers={'X-AppEnlight': abort_msg}) + + +def check_cors(request, application, should_return=True): + """ + Performs a check and validation if request comes from authorized domain for + application, otherwise return 403 + """ + origin_found = False + origin = request.headers.get('Origin') + if should_return: + log.info('CORS for %s' % origin) + if not origin: + return False + for domain in application.domains.split('\n'): + if domain in origin: + origin_found = True + if origin_found: + request.response.headers.add('Access-Control-Allow-Origin', origin) + request.response.headers.add('XDomainRequestAllowed', '1') + request.response.headers.add('Access-Control-Allow-Methods', + 'GET, POST, OPTIONS') + request.response.headers.add('Access-Control-Allow-Headers', + 'Accept-Encoding, Accept-Language, ' + 'Content-Type, ' + 'Depth, User-Agent, X-File-Size, ' + 'X-Requested-With, If-Modified-Since, ' + 'X-File-Name, ' + 'Cache-Control, Host, Pragma, Accept, ' + 'Origin, Connection, ' + 'Referer, Cookie, ' + 'X-appenlight-public-api-key, ' + 'x-appenlight-public-api-key') + request.response.headers.add('Access-Control-Max-Age', '86400') + return request.response + else: + return HTTPForbidden() diff --git a/backend/src/appenlight/lib/cache_regions.py b/backend/src/appenlight/lib/cache_regions.py new file mode 100644 index 0000000..0bdb241 --- /dev/null +++ b/backend/src/appenlight/lib/cache_regions.py @@ -0,0 +1,188 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2010-2016 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# App Enlight Enterprise Edition, including its added features, Support +# services, and proprietary license terms, please see +# https://rhodecode.com/licenses/ + +import copy +import hashlib +import inspect + +from dogpile.cache import make_region, compat + +regions = None + + +def key_mangler(key): + return "appenlight:dogpile:{}".format(key) + + +def hashgen(namespace, fn, to_str=compat.string_type): + """Return a function that generates a string + key, based on a given function as well as + arguments to the returned function itself. + + This is used by :meth:`.CacheRegion.cache_on_arguments` + to generate a cache key from a decorated function. + + It can be replaced using the ``function_key_generator`` + argument passed to :func:`.make_region`. + + """ + + if namespace is None: + namespace = '%s:%s' % (fn.__module__, fn.__name__) + else: + namespace = '%s:%s|%s' % (fn.__module__, fn.__name__, namespace) + + args = inspect.getargspec(fn) + has_self = args[0] and args[0][0] in ('self', 'cls') + + def generate_key(*args, **kw): + if kw: + raise ValueError( + "dogpile.cache's default key creation " + "function does not accept keyword arguments.") + if has_self: + args = args[1:] + + return namespace + "|" + hashlib.sha1( + " ".join(map(to_str, args)).encode('utf8')).hexdigest() + + return generate_key + + +class CacheRegions(object): + def __init__(self, settings): + config_redis = {"arguments": settings} + + self.redis_min_1 = make_region( + function_key_generator=hashgen, + key_mangler=key_mangler).configure( + "dogpile.cache.redis", + expiration_time=60, + **copy.deepcopy(config_redis)) + self.redis_min_5 = make_region( + function_key_generator=hashgen, + key_mangler=key_mangler).configure( + "dogpile.cache.redis", + expiration_time=300, + **copy.deepcopy(config_redis)) + + self.redis_min_10 = make_region( + function_key_generator=hashgen, + key_mangler=key_mangler).configure( + "dogpile.cache.redis", + expiration_time=60, + **copy.deepcopy(config_redis)) + + self.redis_min_60 = make_region( + function_key_generator=hashgen, + key_mangler=key_mangler).configure( + "dogpile.cache.redis", + expiration_time=3600, + **copy.deepcopy(config_redis)) + + self.redis_sec_1 = make_region( + function_key_generator=hashgen, + key_mangler=key_mangler).configure( + "dogpile.cache.redis", + expiration_time=1, + **copy.deepcopy(config_redis)) + + self.redis_sec_5 = make_region( + function_key_generator=hashgen, + key_mangler=key_mangler).configure( + "dogpile.cache.redis", + expiration_time=5, + **copy.deepcopy(config_redis)) + + self.redis_sec_30 = make_region( + function_key_generator=hashgen, + key_mangler=key_mangler).configure( + "dogpile.cache.redis", + expiration_time=30, + **copy.deepcopy(config_redis)) + + self.redis_day_1 = make_region( + function_key_generator=hashgen, + key_mangler=key_mangler).configure( + "dogpile.cache.redis", + expiration_time=86400, + **copy.deepcopy(config_redis)) + + self.redis_day_7 = make_region( + function_key_generator=hashgen, + key_mangler=key_mangler).configure( + "dogpile.cache.redis", + expiration_time=86400 * 7, + **copy.deepcopy(config_redis)) + + self.redis_day_30 = make_region( + function_key_generator=hashgen, + key_mangler=key_mangler).configure( + "dogpile.cache.redis", + expiration_time=86400 * 30, + **copy.deepcopy(config_redis)) + + self.memory_day_1 = make_region( + function_key_generator=hashgen, + key_mangler=key_mangler).configure( + "dogpile.cache.memory", + expiration_time=86400, + **copy.deepcopy(config_redis)) + + self.memory_sec_1 = make_region( + function_key_generator=hashgen, + key_mangler=key_mangler).configure( + "dogpile.cache.memory", + expiration_time=1) + + self.memory_sec_5 = make_region( + function_key_generator=hashgen, + key_mangler=key_mangler).configure( + "dogpile.cache.memory", + expiration_time=5) + + self.memory_min_1 = make_region( + function_key_generator=hashgen, + key_mangler=key_mangler).configure( + "dogpile.cache.memory", + expiration_time=60) + + self.memory_min_5 = make_region( + function_key_generator=hashgen, + key_mangler=key_mangler).configure( + "dogpile.cache.memory", + expiration_time=300) + + self.memory_min_10 = make_region( + function_key_generator=hashgen, + key_mangler=key_mangler).configure( + "dogpile.cache.memory", + expiration_time=600) + + self.memory_min_60 = make_region( + function_key_generator=hashgen, + key_mangler=key_mangler).configure( + "dogpile.cache.memory", + expiration_time=3600) + + +def get_region(region): + return getattr(regions, region) diff --git a/backend/src/appenlight/lib/encryption.py b/backend/src/appenlight/lib/encryption.py new file mode 100644 index 0000000..0f1697d --- /dev/null +++ b/backend/src/appenlight/lib/encryption.py @@ -0,0 +1,63 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2010-2016 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# App Enlight Enterprise Edition, including its added features, Support +# services, and proprietary license terms, please see +# https://rhodecode.com/licenses/ + +# this gets set on runtime +from cryptography.fernet import Fernet + +ENCRYPTION_SECRET = None + + +def encrypt_fernet(value): + # avoid double encryption + # not sure if this is needed but it won't hurt too much to have this + if value.startswith('enc$fernet$'): + return value + f = Fernet(ENCRYPTION_SECRET) + return 'enc$fernet${}'.format(f.encrypt(value.encode('utf8')).decode('utf8')) + + +def decrypt_fernet(value): + parts = value.split('$', 3) + if not len(parts) == 3: + # not encrypted values + return value + else: + f = Fernet(ENCRYPTION_SECRET) + decrypted_data = f.decrypt(parts[2].encode('utf8')).decode('utf8') + return decrypted_data + + +def encrypt_dictionary_keys(_dict, exclude_keys=None): + if not exclude_keys: + exclude_keys = [] + keys = [k for k in _dict.keys() if k not in exclude_keys] + for k in keys: + _dict[k] = encrypt_fernet(_dict[k]) + return _dict + + +def decrypt_dictionary_keys(_dict, exclude_keys=None): + if not exclude_keys: + exclude_keys = [] + keys = [k for k in _dict.keys() if k not in exclude_keys] + for k in keys: + _dict[k] = decrypt_fernet(_dict[k]) + return _dict diff --git a/backend/src/appenlight/lib/enums.py b/backend/src/appenlight/lib/enums.py new file mode 100644 index 0000000..ae4e1eb --- /dev/null +++ b/backend/src/appenlight/lib/enums.py @@ -0,0 +1,93 @@ +import collections +# -*- coding: utf-8 -*- + +# Copyright (C) 2010-2016 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# App Enlight Enterprise Edition, including its added features, Support +# services, and proprietary license terms, please see +# https://rhodecode.com/licenses/ + + +class StupidEnum(object): + @classmethod + def set_inverse(cls): + cls._inverse_values = dict( + (y, x) for x, y in vars(cls).items() if + not x.startswith('_') and not callable(y) + ) + + @classmethod + def key_from_value(cls, value): + if not hasattr(cls, '_inverse_values'): + cls.set_inverse() + return cls._inverse_values.get(value) + + +class ReportType(StupidEnum): + unknown = 0 + error = 1 + not_found = 2 + slow = 3 + + +class Language(StupidEnum): + unknown = 0 + python = 1 + javascript = 2 + java = 3 + objectivec = 4 + swift = 5 + cpp = 6 + basic = 7 + csharp = 8 + php = 9 + perl = 10 + vb = 11 + vbnet = 12 + ruby = 13 + fsharp = 14 + actionscript = 15 + go = 16 + scala = 17 + haskell = 18 + erlang = 19 + haxe = 20 + scheme = 21 + + +class LogLevel(StupidEnum): + UNKNOWN = 0 + DEBUG = 2 + TRACE = 4 + INFO = 6 + WARNING = 8 + ERROR = 10 + CRITICAL = 12 + FATAL = 14 + + +class LogLevelPython(StupidEnum): + CRITICAL = 50 + ERROR = 40 + WARNING = 30 + INFO = 20 + DEBUG = 10 + NOTSET = 0 + + +class ParsedSentryEventType(StupidEnum): + ERROR_REPORT = 1 + LOG = 2 diff --git a/backend/src/appenlight/lib/ext_json.py b/backend/src/appenlight/lib/ext_json.py new file mode 100644 index 0000000..19f7468 --- /dev/null +++ b/backend/src/appenlight/lib/ext_json.py @@ -0,0 +1,153 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2010-2016 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# App Enlight Enterprise Edition, including its added features, Support +# services, and proprietary license terms, please see +# https://rhodecode.com/licenses/ + +""" +ex-json borrowed from Marcin Kuzminski + +source: https://secure.rhodecode.org/ext-json + +""" +import datetime +import functools +import decimal +import imp + +__all__ = ['json', 'simplejson', 'stdlibjson'] + + +def _is_aware(value): + """ + Determines if a given datetime.time is aware. + + The logic is described in Python's docs: + http://docs.python.org/library/datetime.html#datetime.tzinfo + """ + return (value.tzinfo is not None + and value.tzinfo.utcoffset(value) is not None) + + +def _obj_dump(obj): + """ + Custom function for dumping objects to JSON, if obj has __json__ attribute + or method defined it will be used for serialization + + :param obj: + """ + + if isinstance(obj, complex): + return [obj.real, obj.imag] + # See "Date Time String Format" in the ECMA-262 specification. + # some code borrowed from django 1.4 + elif isinstance(obj, datetime.datetime): + r = obj.isoformat() + # if obj.microsecond: + # r = r[:23] + r[26:] + if r.endswith('+00:00'): + r = r[:-6] + 'Z' + return r + elif isinstance(obj, datetime.date): + return obj.isoformat() + elif isinstance(obj, decimal.Decimal): + return str(obj) + elif isinstance(obj, datetime.time): + if _is_aware(obj): + raise ValueError("JSON can't represent timezone-aware times.") + r = obj.isoformat() + if obj.microsecond: + r = r[:12] + return r + elif isinstance(obj, set): + return list(obj) + elif hasattr(obj, '__json__'): + if callable(obj.__json__): + return obj.__json__() + else: + return obj.__json__ + else: + raise NotImplementedError + + +# Import simplejson +try: + # import simplejson initially + _sj = imp.load_module('_sj', *imp.find_module('simplejson')) + + + def extended_encode(obj): + try: + return _obj_dump(obj) + except NotImplementedError: + pass + raise TypeError("%r is not JSON serializable" % (obj,)) + + + # we handle decimals our own it makes unified behavior of json vs + # simplejson + sj_version = [int(x) for x in _sj.__version__.split('.')] + major, minor = sj_version[0], sj_version[1] + if major < 2 or (major == 2 and minor < 1): + # simplejson < 2.1 doesnt support use_decimal + _sj.dumps = functools.partial( + _sj.dumps, default=extended_encode) + _sj.dump = functools.partial( + _sj.dump, default=extended_encode) + else: + _sj.dumps = functools.partial( + _sj.dumps, default=extended_encode, use_decimal=False) + _sj.dump = functools.partial( + _sj.dump, default=extended_encode, use_decimal=False) + simplejson = _sj + +except ImportError: + # no simplejson set it to None + simplejson = None + +try: + # simplejson not found try out regular json module + _json = imp.load_module('_json', *imp.find_module('json')) + + + # extended JSON encoder for json + class ExtendedEncoder(_json.JSONEncoder): + def default(self, obj): + try: + return _obj_dump(obj) + except NotImplementedError: + pass + raise TypeError("%r is not JSON serializable" % (obj,)) + + + # monkey-patch JSON encoder to use extended version + _json.dumps = functools.partial(_json.dumps, cls=ExtendedEncoder) + _json.dump = functools.partial(_json.dump, cls=ExtendedEncoder) + +except ImportError: + json = None + +stdlibjson = _json + +# set all available json modules +if simplejson: + json = _sj +elif _json: + json = _json +else: + raise ImportError('Could not find any json modules') diff --git a/backend/src/appenlight/lib/helpers.py b/backend/src/appenlight/lib/helpers.py new file mode 100644 index 0000000..e48e7c6 --- /dev/null +++ b/backend/src/appenlight/lib/helpers.py @@ -0,0 +1,124 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2010-2016 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# App Enlight Enterprise Edition, including its added features, Support +# services, and proprietary license terms, please see +# https://rhodecode.com/licenses/ + +""" +Helper functions +""" +import copy +import datetime + +from collections import namedtuple, OrderedDict + +_ = lambda x: x + +time_deltas = OrderedDict() + +time_deltas['1m'] = {'delta': datetime.timedelta(minutes=1), + 'label': '1 minute', 'minutes': 1} + +time_deltas['5m'] = {'delta': datetime.timedelta(minutes=5), + 'label': '5 minutes', 'minutes': 5} +time_deltas['30m'] = {'delta': datetime.timedelta(minutes=30), + 'label': '30 minutes', 'minutes': 30} +time_deltas['1h'] = {'delta': datetime.timedelta(hours=1), + 'label': '60 minutes', 'minutes': 60} +time_deltas['4h'] = {'delta': datetime.timedelta(hours=4), 'label': '4 hours', + 'minutes': 60 * 4} +time_deltas['12h'] = {'delta': datetime.timedelta(hours=12), + 'label': '12 hours', 'minutes': 60 * 12} +time_deltas['24h'] = {'delta': datetime.timedelta(hours=24), + 'label': '24 hours', 'minutes': 60 * 24} +time_deltas['3d'] = {'delta': datetime.timedelta(days=3), 'label': '3 days', + 'minutes': 60 * 24 * 3} +time_deltas['1w'] = {'delta': datetime.timedelta(days=7), 'label': '7 days', + 'minutes': 60 * 24 * 7} +time_deltas['2w'] = {'delta': datetime.timedelta(days=14), 'label': '14 days', + 'minutes': 60 * 24 * 14} +time_deltas['1M'] = {'delta': datetime.timedelta(days=31), 'label': '31 days', + 'minutes': 60 * 24 * 31} +time_deltas['3M'] = {'delta': datetime.timedelta(days=31 * 3), + 'label': '3 months', + 'minutes': 60 * 24 * 31 * 3} +time_deltas['6M'] = {'delta': datetime.timedelta(days=31 * 6), + 'label': '6 months', + 'minutes': 60 * 24 * 31 * 6} +time_deltas['12M'] = {'delta': datetime.timedelta(days=31 * 12), + 'label': '12 months', + 'minutes': 60 * 24 * 31 * 12} + +# used in json representation +time_options = dict([(k, {'label': v['label'], 'minutes': v['minutes']}) + for k, v in time_deltas.items()]) +FlashMsg = namedtuple('FlashMsg', ['msg', 'level']) + + +def get_flash(request): + messages = [] + messages.extend( + [FlashMsg(msg, 'error') + for msg in request.session.peek_flash('error')]) + messages.extend([FlashMsg(msg, 'warning') + for msg in request.session.peek_flash('warning')]) + messages.extend( + [FlashMsg(msg, 'notice') for msg in request.session.peek_flash()]) + return messages + + +def clear_flash(request): + request.session.pop_flash('error') + request.session.pop_flash('warning') + request.session.pop_flash() + + +def get_type_formatted_flash(request): + return [{'msg': message.msg, 'type': message.level} + for message in get_flash(request)] + + +def gen_pagination_headers(request, paginator): + headers = { + 'x-total-count': str(paginator.item_count), + 'x-current-page': str(paginator.page), + 'x-items-per-page': str(paginator.items_per_page) + } + params_dict = request.GET.dict_of_lists() + last_page_params = copy.deepcopy(params_dict) + last_page_params['page'] = paginator.last_page or 1 + first_page_params = copy.deepcopy(params_dict) + first_page_params.pop('page', None) + next_page_params = copy.deepcopy(params_dict) + next_page_params['page'] = paginator.next_page or paginator.last_page or 1 + prev_page_params = copy.deepcopy(params_dict) + prev_page_params['page'] = paginator.previous_page or 1 + lp_url = request.current_route_url(_query=last_page_params) + fp_url = request.current_route_url(_query=first_page_params) + links = [ + 'rel="last", <{}>'.format(lp_url), + 'rel="first", <{}>'.format(fp_url), + ] + if first_page_params != prev_page_params: + prev_url = request.current_route_url(_query=prev_page_params) + links.append('rel="prev", <{}>'.format(prev_url)) + if last_page_params != next_page_params: + next_url = request.current_route_url(_query=next_page_params) + links.append('rel="next", <{}>'.format(next_url)) + headers['link'] = '; '.join(links) + return headers diff --git a/backend/src/appenlight/lib/jinja2_filters.py b/backend/src/appenlight/lib/jinja2_filters.py new file mode 100644 index 0000000..82dc592 --- /dev/null +++ b/backend/src/appenlight/lib/jinja2_filters.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2010-2016 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# App Enlight Enterprise Edition, including its added features, Support +# services, and proprietary license terms, please see +# https://rhodecode.com/licenses/ + +import re +from appenlight.lib.ext_json import json +from jinja2 import Markup, escape, evalcontextfilter + +_paragraph_re = re.compile(r'(?:\r\n|\r|\n){2,}') + + +@evalcontextfilter +def nl2br(eval_ctx, value): + if eval_ctx.autoescape: + result = '\n\n'.join('

%s

' % p.replace('\n', Markup('
\n')) + for p in _paragraph_re.split(escape(value))) + else: + result = '\n\n'.join('

%s

' % p.replace('\n', '
\n') + for p in _paragraph_re.split(escape(value))) + if eval_ctx.autoescape: + result = Markup(result) + return result + + +@evalcontextfilter +def toJSONUnsafe(eval_ctx, value): + encoded = json.dumps(value).replace('&', '\\u0026') \ + .replace('<', '\\u003c') \ + .replace('>', '\\u003e') \ + .replace('>', '\\u003e') \ + .replace('"', '\\u0022') \ + .replace("'", '\\u0027') \ + .replace(r'\n', '/\\\n') + return Markup("'%s'" % encoded) diff --git a/backend/src/appenlight/lib/redis_keys.py b/backend/src/appenlight/lib/redis_keys.py new file mode 100644 index 0000000..de0e2a3 --- /dev/null +++ b/backend/src/appenlight/lib/redis_keys.py @@ -0,0 +1,59 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2010-2016 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# App Enlight Enterprise Edition, including its added features, Support +# services, and proprietary license terms, please see +# https://rhodecode.com/licenses/ + +BASE = 'appenlight:data:{}' + +REDIS_KEYS = { + 'tasks': { + 'add_reports_lock': BASE.format('add_reports_lock:{}'), + 'add_logs_lock': BASE.format('add_logs_lock:{}'), + }, + 'counters': { + 'reports_per_minute': BASE.format('reports_per_minute:{}'), + 'reports_per_minute_per_app': BASE.format( + 'reports_per_minute_per_app:{}:{}'), + 'reports_per_type': BASE.format('reports_per_type:{}'), + 'logs_per_minute': BASE.format('logs_per_minute:{}'), + 'logs_per_minute_per_app': BASE.format( + 'logs_per_minute_per_app:{}:{}'), + 'metrics_per_minute': BASE.format('metrics_per_minute:{}'), + 'metrics_per_minute_per_app': BASE.format( + 'metrics_per_minute_per_app:{}:{}'), + 'report_group_occurences': BASE.format('report_group_occurences:{}'), + 'report_group_occurences_10th': BASE.format( + 'report_group_occurences_10th:{}'), + 'report_group_occurences_100th': BASE.format( + 'report_group_occurences_100th:{}'), + }, + 'rate_limits': { + 'per_application_reports_rate_limit': BASE.format( + 'per_application_reports_limit:{}:{}'), + 'per_application_logs_rate_limit': BASE.format( + 'per_application_logs_rate_limit:{}:{}'), + 'per_application_metrics_rate_limit': BASE.format( + 'per_application_metrics_rate_limit:{}:{}'), + }, + 'apps_that_had_reports': BASE.format('apps_that_had_reports'), + 'apps_that_had_error_reports': BASE.format('apps_that_had_error_reports'), + 'reports_to_notify_per_type_per_app': BASE.format( + 'reports_to_notify_per_type_per_app:{}:{}'), + 'seen_tag_list': BASE.format('seen_tag_list') +} diff --git a/backend/src/appenlight/lib/request.py b/backend/src/appenlight/lib/request.py new file mode 100644 index 0000000..cb91ea0 --- /dev/null +++ b/backend/src/appenlight/lib/request.py @@ -0,0 +1,89 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2010-2016 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# App Enlight Enterprise Edition, including its added features, Support +# services, and proprietary license terms, please see +# https://rhodecode.com/licenses/ + +import appenlight.lib.helpers as helpers +import json +from pyramid.security import unauthenticated_userid +from appenlight.models.user import User + + +class CSRFException(Exception): + pass + + +class JSONException(Exception): + pass + + +def get_csrf_token(request): + return request.session.get_csrf_token() + + +def safe_json_body(request): + """ + Returns None if json body is missing or erroneous + """ + try: + return request.json_body + except ValueError: + return None + + +def unsafe_json_body(request): + """ + Throws JSONException if json can't deserialize + """ + try: + return request.json_body + except ValueError: + raise JSONException('Incorrect JSON') + + +def get_user(request): + if not request.path_info.startswith('/static'): + user_id = unauthenticated_userid(request) + try: + user_id = int(user_id) + except Exception: + return None + + if user_id: + user = User.by_id(user_id) + if user: + request.environ['appenlight.username'] = '%d:%s' % ( + user_id, user.user_name) + return user + else: + return None + + +def es_conn(request): + return request.registry.es_conn + + +def add_flash_to_headers(request, clear=True): + """ + Adds pending flash messages to response, if clear is true clears out the + flash queue + """ + flash_msgs = helpers.get_type_formatted_flash(request) + request.response.headers['x-flash-messages'] = json.dumps(flash_msgs) + helpers.clear_flash(request) diff --git a/backend/src/appenlight/lib/rule.py b/backend/src/appenlight/lib/rule.py new file mode 100644 index 0000000..a9de9fc --- /dev/null +++ b/backend/src/appenlight/lib/rule.py @@ -0,0 +1,288 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2010-2016 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# App Enlight Enterprise Edition, including its added features, Support +# services, and proprietary license terms, please see +# https://rhodecode.com/licenses/ + +import logging +import operator + +log = logging.getLogger(__name__) + + +class RuleException(Exception): + pass + + +class KeyNotFoundException(RuleException): + pass + + +class UnknownTypeException(RuleException): + pass + + +class BadConfigException(RuleException): + pass + + +class InvalidValueException(RuleException): + pass + + +class RuleBase(object): + @classmethod + def default_dict_struct_getter(cls, struct, field_name): + """ + returns a key from dictionary based on field_name, if the name contains + `:` then it means additional nesting levels should be checked for the + key so `a:b:c` means return struct['a']['b']['c'] + + :param struct: + :param field_name: + :return: + """ + parts = field_name.split(':') if field_name else [] + found = struct + while parts: + current_key = parts.pop(0) + found = found.get(current_key) + if not found and parts: + raise KeyNotFoundException('Key not found in structure') + return found + + @classmethod + def default_obj_struct_getter(cls, struct, field_name): + """ + returns a key from instance based on field_name, if the name contains + `:` then it means additional nesting levels should be checked for the + key so `a:b:c` means return struct.a.b.c + + :param struct: + :param field_name: + :return: + """ + parts = field_name.split(':') + found = struct + while parts: + current_key = parts.pop(0) + found = getattr(found, current_key, None) + if not found and parts: + raise KeyNotFoundException('Key not found in structure') + return found + + def normalized_type(self, field, value): + """ + Converts text values from self.conf_value based on type_matrix below + check_matrix defines what kind of checks we can perform on a field + value based on field name + """ + f_type = self.type_matrix.get(field) + if f_type: + cast_to = f_type['type'] + else: + raise UnknownTypeException('Unknown type') + + if value is None: + return None + + try: + if cast_to == 'int': + return int(value) + elif cast_to == 'float': + return float(value) + elif cast_to == 'unicode': + return str(value) + except ValueError as exc: + raise InvalidValueException(exc) + + +class Rule(RuleBase): + def __init__(self, config, type_matrix, + struct_getter=RuleBase.default_dict_struct_getter, + config_manipulator=None): + """ + + :param config: dict - contains rule configuration + example:: + { + "field": "__OR__", + "rules": [ + { + "field": "__AND__", + "rules": [ + { + "op": "ge", + "field": "occurences", + "value": "10" + }, + { + "op": "ge", + "field": "priority", + "value": "4" + } + ] + }, + { + "op": "eq", + "field": "http_status", + "value": "500" + } + ] + } + :param type_matrix: dict - contains map of type casts + example:: + { + 'http_status': 'int', + 'priority': 'unicode', + } + :param struct_getter: callable - used to grab the value of field from + the structure passed to match() based + on key, default + + """ + self.type_matrix = type_matrix + self.config = config + self.struct_getter = struct_getter + self.config_manipulator = config_manipulator + if config_manipulator: + config_manipulator(self) + + def subrule_check(self, rule_config, struct): + rule = Rule(rule_config, self.type_matrix, + config_manipulator=self.config_manipulator) + return rule.match(struct) + + def match(self, struct): + """ + Check if rule matched for this specific report + First tries report value, then tests tags in not found, then finally + report group + """ + field_name = self.config.get('field') + test_value = self.config.get('value') + + if not field_name: + return False + + if field_name == '__AND__': + rule = AND(self.config['rules'], self.type_matrix, + config_manipulator=self.config_manipulator) + return rule.match(struct) + elif field_name == '__OR__': + rule = OR(self.config['rules'], self.type_matrix, + config_manipulator=self.config_manipulator) + return rule.match(struct) + + if test_value is None: + return False + + try: + struct_value = self.normalized_type(field_name, + self.struct_getter(struct, + field_name)) + except (UnknownTypeException, InvalidValueException) as exc: + log.error(str(exc)) + return False + + try: + test_value = self.normalized_type(field_name, test_value) + except (UnknownTypeException, InvalidValueException) as exc: + log.error(str(exc)) + return False + + if self.config['op'] not in ('startswith', 'endswith', 'contains'): + try: + return getattr(operator, + self.config['op'])(struct_value, test_value) + except TypeError: + return False + elif self.config['op'] == 'startswith': + return struct_value.startswith(test_value) + elif self.config['op'] == 'endswith': + return struct_value.endswith(test_value) + elif self.config['op'] == 'contains': + return test_value in struct_value + raise BadConfigException('Invalid configuration, ' + 'unknown operator: {}'.format(self.config)) + + def __repr__(self): + return ''.format(self.config.get('field'), + self.config.get('value')) + + +class AND(Rule): + def __init__(self, rules, *args, **kwargs): + super(AND, self).__init__({}, *args, **kwargs) + self.rules = rules + + def match(self, struct): + return all([self.subrule_check(r_conf, struct) for r_conf + in self.rules]) + + +class OR(Rule): + def __init__(self, rules, *args, **kwargs): + super(OR, self).__init__({}, *args, **kwargs) + self.rules = rules + + def match(self, struct): + return any([self.subrule_check(r_conf, struct) for r_conf + in self.rules]) + + +class RuleService(object): + @staticmethod + def rule_from_config(config, field_mappings, labels_dict, + manipulator_func=None): + """ + Returns modified rule with manipulator function + By default manipulator function replaces field id from labels_dict + with current field id proper for the rule from fields_mappings + + because label X_X id might be pointing different value on next request + when new term is returned from elasticsearch - this ensures things + are kept 1:1 all the time + """ + rev_map = {} + for k, v in labels_dict.items(): + rev_map[(v['agg'], v['key'],)] = k + + if manipulator_func is None: + def label_rewriter_func(rule): + field = rule.config.get('field') + if not field or rule.config['field'] in ['__OR__', '__AND__']: + return + + to_map = field_mappings.get(rule.config['field']) + + # we need to replace series field with _AE_NOT_FOUND_ to not match + # accidently some other field which happens to have the series that + # was used when the alert was created + if to_map: + to_replace = rev_map.get((to_map['agg'], to_map['key'],), + '_AE_NOT_FOUND_') + else: + to_replace = '_AE_NOT_FOUND_' + + rule.config['field'] = to_replace + rule.type_matrix[to_replace] = {"type": 'float'} + + manipulator_func = label_rewriter_func + + return Rule(config, {}, config_manipulator=manipulator_func) diff --git a/backend/src/appenlight/lib/social.py b/backend/src/appenlight/lib/social.py new file mode 100644 index 0000000..b1c578c --- /dev/null +++ b/backend/src/appenlight/lib/social.py @@ -0,0 +1,65 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2010-2016 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# App Enlight Enterprise Edition, including its added features, Support +# services, and proprietary license terms, please see +# https://rhodecode.com/licenses/ + +from ziggurat_foundations.models.services.external_identity import \ + ExternalIdentityService +from appenlight.models.external_identity import ExternalIdentity + + +def handle_social_data(request, user, social_data): + social_data = social_data + update_identity = False + + extng_id = ExternalIdentityService.by_external_id_and_provider( + social_data['user']['id'], + social_data['credentials'].provider_name + ) + + # fix legacy accounts with wrong google ID + if not extng_id and social_data['credentials'].provider_name == 'google': + extng_id = ExternalIdentityService.by_external_id_and_provider( + social_data['user']['email'], + social_data['credentials'].provider_name + ) + + if extng_id: + extng_id.delete() + update_identity = True + + if not social_data['user']['id']: + request.session.flash( + 'No external user id found? Perhaps permissions for ' + 'authentication are set incorrectly', 'error') + return False + + if not extng_id or update_identity: + if not update_identity: + request.session.flash('Your external identity is now ' + 'connected with your account') + ex_identity = ExternalIdentity() + ex_identity.external_id = social_data['user']['id'] + ex_identity.external_user_name = social_data['user']['user_name'] + ex_identity.provider_name = social_data['credentials'].provider_name + ex_identity.access_token = social_data['credentials'].token + ex_identity.token_secret = social_data['credentials'].token_secret + ex_identity.alt_token = social_data['credentials'].refresh_token + user.external_identities.append(ex_identity) + request.session.pop('zigg.social_auth', None) diff --git a/backend/src/appenlight/lib/sqlalchemy_fields.py b/backend/src/appenlight/lib/sqlalchemy_fields.py new file mode 100644 index 0000000..21405fb --- /dev/null +++ b/backend/src/appenlight/lib/sqlalchemy_fields.py @@ -0,0 +1,54 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2010-2016 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# App Enlight Enterprise Edition, including its added features, Support +# services, and proprietary license terms, please see +# https://rhodecode.com/licenses/ + +import binascii +import sqlalchemy.types as types + +import appenlight.lib.encryption as encryption + + +class BinaryHex(types.TypeDecorator): + impl = types.LargeBinary + + def process_bind_param(self, value, dialect): + if value is not None: + value = binascii.unhexlify(value) + + return value + + def process_result_value(self, value, dialect): + if value is not None: + value = binascii.hexlify(value) + return value + + +class EncryptedUnicode(types.TypeDecorator): + impl = types.Unicode + + def process_bind_param(self, value, dialect): + if not value: + return value + return encryption.encrypt_fernet(value) + + def process_result_value(self, value, dialect): + if not value: + return value + return encryption.decrypt_fernet(value) diff --git a/backend/src/appenlight/lib/utils/__init__.py b/backend/src/appenlight/lib/utils/__init__.py new file mode 100644 index 0000000..b873788 --- /dev/null +++ b/backend/src/appenlight/lib/utils/__init__.py @@ -0,0 +1,495 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2010-2016 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# App Enlight Enterprise Edition, including its added features, Support +# services, and proprietary license terms, please see +# https://rhodecode.com/licenses/ + +""" +Utility functions. +""" +import logging +import requests +import hashlib +import json +import copy +import uuid +import appenlight.lib.helpers as h +from collections import namedtuple +from datetime import timedelta, datetime, date +from dogpile.cache.api import NO_VALUE +from appenlight.models import Datastores +from appenlight.validators import (LogSearchSchema, + TagListSchema, + accepted_search_params) +from itsdangerous import TimestampSigner +from ziggurat_foundations.permissions import ALL_PERMISSIONS +from dateutil.relativedelta import relativedelta +from dateutil.rrule import rrule, MONTHLY, DAILY + +log = logging.getLogger(__name__) + + +Stat = namedtuple('Stat', 'start_interval value') + + +def default_extractor(item): + """ + :param item - item to extract date from + """ + if hasattr(item, 'start_interval'): + return item.start_interval + return item['start_interval'] + + +# fast gap generator +def gap_gen_default(start, step, itemiterator, end_time=None, + iv_extractor=None): + """ generates a list of time/value items based on step and itemiterator + if there are entries missing from iterator time/None will be returned + instead + :param start - datetime - what time should we start generating our values + :param step - timedelta - stepsize + :param itemiterator - iterable - we will check this iterable for values + corresponding to generated steps + :param end_time - datetime - when last step is >= end_time stop iterating + :param iv_extractor - extracts current step from iterable items + """ + + if not iv_extractor: + iv_extractor = default_extractor + + next_step = start + minutes = step.total_seconds() / 60.0 + while next_step.minute % minutes != 0: + next_step = next_step.replace(minute=next_step.minute - 1) + for item in itemiterator: + item_start_interval = iv_extractor(item) + # do we have a match for current time step in our data? + # no gen a new tuple with 0 values + while next_step < item_start_interval: + yield Stat(next_step, None) + next_step = next_step + step + if next_step == item_start_interval: + yield Stat(item_start_interval, item) + next_step = next_step + step + if end_time: + while next_step < end_time: + yield Stat(next_step, None) + next_step = next_step + step + + +class DateTimeEncoder(json.JSONEncoder): + """ Simple datetime to ISO encoder for json serialization""" + + def default(self, obj): + if isinstance(obj, date): + return obj.isoformat() + if isinstance(obj, datetime): + return obj.isoformat() + return json.JSONEncoder.default(self, obj) + + +def cometd_request(secret, endpoint, payload, throw_exceptions=False, + servers=None): + responses = [] + if not servers: + servers = [] + + signer = TimestampSigner(secret) + sig_for_server = signer.sign(endpoint) + for secret, server in [(s['secret'], s['server']) for s in servers]: + response = {} + secret_headers = {'x-channelstream-secret': sig_for_server, + 'x-channelstream-endpoint': endpoint, + 'Content-Type': 'application/json'} + url = '%s%s' % (server, endpoint) + try: + response = requests.post(url, + data=json.dumps(payload, + cls=DateTimeEncoder), + headers=secret_headers, + verify=False, + timeout=2).json() + except requests.exceptions.RequestException as e: + if throw_exceptions: + raise + responses.append(response) + return responses + + +def add_cors_headers(response): + # allow CORS + response.headers.add('Access-Control-Allow-Origin', '*') + response.headers.add('XDomainRequestAllowed', '1') + response.headers.add('Access-Control-Allow-Methods', 'GET, POST, OPTIONS') + # response.headers.add('Access-Control-Allow-Credentials', 'true') + response.headers.add('Access-Control-Allow-Headers', + 'Content-Type, Depth, User-Agent, X-File-Size, X-Requested-With, If-Modified-Since, X-File-Name, Cache-Control, Pragma, Origin, Connection, Referer, Cookie') + response.headers.add('Access-Control-Max-Age', '86400') + + +from sqlalchemy.sql import compiler +from psycopg2.extensions import adapt as sqlescape + + +# or use the appropiate escape function from your db driver + +def compile_query(query): + dialect = query.session.bind.dialect + statement = query.statement + comp = compiler.SQLCompiler(dialect, statement) + comp.compile() + enc = dialect.encoding + params = {} + for k, v in comp.params.items(): + if isinstance(v, str): + v = v.encode(enc) + params[k] = sqlescape(v) + return (comp.string.encode(enc) % params).decode(enc) + + +def convert_es_type(input_data): + """ + This might need to convert some text or other types to corresponding ES types + """ + return str(input_data) + + +ProtoVersion = namedtuple('ProtoVersion', ['major', 'minor', 'patch']) + + +def parse_proto(input_data): + try: + parts = [int(x) for x in input_data.split('.')] + while len(parts) < 3: + parts.append(0) + return ProtoVersion(*parts) + except Exception as e: + log.info('Unknown protocol version: %s' % e) + return ProtoVersion(99, 99, 99) + + +def es_index_name_limiter(start_date=None, end_date=None, months_in_past=6, + ixtypes=None): + """ + This function limits the search to 6 months by default so we don't have to + query 300 elasticsearch indices for 20 years of historical data for example + """ + + # should be cached later + def get_possible_names(): + return list(Datastores.es.aliases().keys()) + + possible_names = get_possible_names() + es_index_types = [] + if not ixtypes: + ixtypes = ['reports', 'metrics', 'logs'] + for t in ixtypes: + if t == 'reports': + es_index_types.append('rcae_r_%s') + elif t == 'logs': + es_index_types.append('rcae_l_%s') + elif t == 'metrics': + es_index_types.append('rcae_m_%s') + elif t == 'uptime': + es_index_types.append('rcae_u_%s') + elif t == 'slow_calls': + es_index_types.append('rcae_sc_%s') + + if start_date: + start_date = copy.copy(start_date) + else: + if not end_date: + end_date = datetime.utcnow() + start_date = end_date + relativedelta(months=months_in_past * -1) + + if not end_date: + end_date = start_date + relativedelta(months=months_in_past) + + index_dates = list(rrule(MONTHLY, + dtstart=start_date.date().replace(day=1), + until=end_date.date(), + count=36)) + index_names = [] + for ix_type in es_index_types: + to_extend = [ix_type % d.strftime('%Y_%m') for d in index_dates + if ix_type % d.strftime('%Y_%m') in possible_names] + index_names.extend(to_extend) + for day in list(rrule(DAILY, dtstart=start_date.date(), + until=end_date.date(), count=366)): + ix_name = ix_type % day.strftime('%Y_%m_%d') + if ix_name in possible_names: + index_names.append(ix_name) + return index_names + + +def build_filter_settings_from_query_dict( + request, params=None, override_app_ids=None, + resource_permissions=None): + """ + Builds list of normalized search terms for ES from query params + ensuring application list is restricted to only applications user + has access to + + :param params (dictionary) + :param override_app_ids - list of application id's to use instead of + applications user normally has access to + """ + params = copy.deepcopy(params) + applications = [] + if not resource_permissions: + resource_permissions = ['view'] + + if request.user: + applications = request.user.resources_with_perms( + resource_permissions, resource_types=['application']) + + # CRITICAL - this ensures our resultset is limited to only the ones + # user has view permissions + all_possible_app_ids = set([app.resource_id for app in applications]) + + # if override is preset we force permission for app to be present + # this allows users to see dashboards and applications they would + # normally not be able to + + if override_app_ids: + all_possible_app_ids = set(override_app_ids) + + schema = LogSearchSchema().bind(resources=all_possible_app_ids) + tag_schema = TagListSchema() + filter_settings = schema.deserialize(params) + tag_list = [] + for k, v in list(filter_settings.items()): + if k in accepted_search_params: + continue + tag_list.append({"name": k, "value": v, "op": 'eq'}) + # remove the key from filter_settings + filter_settings.pop(k, None) + tags = tag_schema.deserialize(tag_list) + filter_settings['tags'] = tags + return filter_settings + + +def gen_uuid(): + return str(uuid.uuid4()) + + +def gen_uuid4_sha_hex(): + return hashlib.sha1(uuid.uuid4().bytes).hexdigest() + + +def permission_tuple_to_dict(data): + out = { + "user_name": None, + "perm_name": data.perm_name, + "owner": data.owner, + "type": data.type, + "resource_name": None, + "resource_type": None, + "resource_id": None, + "group_name": None, + "group_id": None + } + if data.user: + out["user_name"] = data.user.user_name + if data.perm_name == ALL_PERMISSIONS: + out['perm_name'] = '__all_permissions__' + if data.resource: + out['resource_name'] = data.resource.resource_name + out['resource_type'] = data.resource.resource_type + out['resource_id'] = data.resource.resource_id + if data.group: + out['group_name'] = data.group.group_name + out['group_id'] = data.group.id + return out + + +def get_cached_buckets(request, stats_since, end_time, fn, cache_key, + gap_gen=None, db_session=None, step_interval=None, + iv_extractor=None, + rerange=False, *args, **kwargs): + """ Takes "fn" that should return some data and tries to load the data + dividing it into daily buckets - if the stats_since and end time give a + delta bigger than 24hours, then only "todays" data is computed on the fly + + :param request: (request) request object + :param stats_since: (datetime) start date of buckets range + :param end_time: (datetime) end date of buckets range - utcnow() if None + :param fn: (callable) callable to use to populate buckets should have + following signature: + def get_data(request, since_when, until, *args, **kwargs): + + :param cache_key: (string) cache key that will be used to build bucket + caches + :param gap_gen: (callable) gap generator - should return step intervals + to use with out `fn` callable + :param db_session: (Session) sqlalchemy session + :param step_interval: (timedelta) optional step interval if we want to + override the default determined from total start/end time delta + :param iv_extractor: (callable) used to get step intervals from data + returned by `fn` callable + :param rerange: (bool) handy if we want to change ranges from hours to + days when cached data is missing - will shorten execution time if `fn` + callable supports that and we are working with multiple rows - like metrics + :param args: + :param kwargs: + + :return: iterable + """ + if not end_time: + end_time = datetime.utcnow().replace(second=0, microsecond=0) + delta = end_time - stats_since + # if smaller than 3 days we want to group by 5min else by 1h, + # for 60 min group by min + if not gap_gen: + gap_gen = gap_gen_default + if not iv_extractor: + iv_extractor = default_extractor + + # do not use custom interval if total time range with new iv would exceed + # end time + if not step_interval or stats_since + step_interval >= end_time: + if delta < h.time_deltas.get('12h')['delta']: + step_interval = timedelta(seconds=60) + elif delta < h.time_deltas.get('3d')['delta']: + step_interval = timedelta(seconds=60 * 5) + elif delta > h.time_deltas.get('2w')['delta']: + step_interval = timedelta(days=1) + else: + step_interval = timedelta(minutes=60) + + if step_interval >= timedelta(minutes=60): + log.info('cached_buckets:{}: adjusting start time ' + 'for hourly or daily intervals'.format(cache_key)) + stats_since = stats_since.replace(hour=0, minute=0) + + ranges = [i.start_interval for i in list(gap_gen(stats_since, + step_interval, [], + end_time=end_time))] + buckets = {} + storage_key = 'buckets:' + cache_key + '{}|{}' + # this means we basicly cache per hour in 3-14 day intervals but i think + # its fine at this point - will be faster than db access anyways + + if len(ranges) >= 1: + last_ranges = [ranges[-1]] + else: + last_ranges = [] + if step_interval >= timedelta(minutes=60): + for r in ranges: + k = storage_key.format(step_interval.total_seconds(), r) + value = request.registry.cache_regions.redis_day_30.get(k) + # last buckets are never loaded from cache + is_last_result = ( + r >= end_time - timedelta(hours=6) or r in last_ranges) + if value is not NO_VALUE and not is_last_result: + log.info("cached_buckets:{}: " + "loading range {} from cache".format(cache_key, r)) + buckets[r] = value + else: + log.info("cached_buckets:{}: " + "loading range {} from storage".format(cache_key, r)) + range_size = step_interval + if (step_interval == timedelta(minutes=60) and + not is_last_result and rerange): + range_size = timedelta(days=1) + r = r.replace(hour=0, minute=0) + log.info("cached_buckets:{}: " + "loading collapsed " + "range {} {}".format(cache_key, r, + r + range_size)) + bucket_data = fn( + request, r, r + range_size, step_interval, + gap_gen, bucket_count=len(ranges), *args, **kwargs) + for b in bucket_data: + b_iv = iv_extractor(b) + buckets[b_iv] = b + k2 = storage_key.format( + step_interval.total_seconds(), b_iv) + request.registry.cache_regions.redis_day_30.set(k2, b) + log.info("cached_buckets:{}: saving cache".format(cache_key)) + else: + # bucket count is 1 for short time ranges <= 24h from now + bucket_data = fn(request, stats_since, end_time, step_interval, + gap_gen, bucket_count=1, *args, **kwargs) + for b in bucket_data: + buckets[iv_extractor(b)] = b + return buckets + + +def get_cached_split_data(request, stats_since, end_time, fn, cache_key, + db_session=None, *args, **kwargs): + """ Takes "fn" that should return some data and tries to load the data + dividing it into 2 buckets - cached "since_from" bucket and "today" + bucket - then the data can be reduced into single value + + Data is cached if the stats_since and end time give a delta bigger + than 24hours - then only 24h is computed on the fly + """ + if not end_time: + end_time = datetime.utcnow().replace(second=0, microsecond=0) + delta = end_time - stats_since + + if delta >= timedelta(minutes=60): + log.info('cached_split_data:{}: adjusting start time ' + 'for hourly or daily intervals'.format(cache_key)) + stats_since = stats_since.replace(hour=0, minute=0) + + storage_key = 'buckets_split_data:' + cache_key + ':{}|{}' + old_end_time = end_time.replace(hour=0, minute=0) + + final_storage_key = storage_key.format(delta.total_seconds(), + old_end_time) + older_data = None + + cdata = request.registry.cache_regions.redis_day_7.get( + final_storage_key) + + if cdata: + log.info("cached_split_data:{}: found old " + "bucket data".format(cache_key)) + older_data = cdata + + if (stats_since < end_time - h.time_deltas.get('24h')['delta'] and + not cdata): + log.info("cached_split_data:{}: didn't find the " + "start bucket in cache so load older data".format(cache_key)) + recent_stats_since = old_end_time + older_data = fn(request, stats_since, recent_stats_since, + db_session=db_session, *args, **kwargs) + request.registry.cache_regions.redis_day_7.set(final_storage_key, + older_data) + elif stats_since < end_time - h.time_deltas.get('24h')['delta']: + recent_stats_since = old_end_time + else: + recent_stats_since = stats_since + + log.info("cached_split_data:{}: loading fresh " + "data bucksts from last 24h ".format(cache_key)) + todays_data = fn(request, recent_stats_since, end_time, + db_session=db_session, *args, **kwargs) + return older_data, todays_data + + +def in_batches(seq, size): + """ + Splits am iterable into batches of specified size + :param seq (iterable) + :param size integer + """ + return (seq[pos:pos + size] for pos in range(0, len(seq), size)) diff --git a/backend/src/appenlight/lib/utils/airbrake.py b/backend/src/appenlight/lib/utils/airbrake.py new file mode 100644 index 0000000..82870f7 --- /dev/null +++ b/backend/src/appenlight/lib/utils/airbrake.py @@ -0,0 +1,147 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2010-2016 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# App Enlight Enterprise Edition, including its added features, Support +# services, and proprietary license terms, please see +# https://rhodecode.com/licenses/ + +import logging +import uuid + +from datetime import datetime + +log = logging.getLogger(__name__) + + +def parse_airbrake_xml(request): + root = request.context.airbrake_xml_etree + error = root.find('error') + notifier = root.find('notifier') + server_env = root.find('server-environment') + request_data = root.find('request') + user = root.find('current-user') + if request_data is not None: + cgi_data = request_data.find('cgi-data') + if cgi_data is None: + cgi_data = [] + + error_dict = { + 'class_name': error.findtext('class') or '', + 'error': error.findtext('message') or '', + "occurences": 1, + "http_status": 500, + "priority": 5, + "server": 'unknown', + 'url': 'unknown', 'request': {} + } + if user is not None: + error_dict['username'] = user.findtext('username') or \ + user.findtext('id') + if notifier is not None: + error_dict['client'] = notifier.findtext('name') + + if server_env is not None: + error_dict["server"] = server_env.findtext('hostname', 'unknown') + + whitelist_environ = ['REMOTE_USER', 'REMOTE_ADDR', 'SERVER_NAME', + 'CONTENT_TYPE', 'HTTP_REFERER'] + + if request_data is not None: + error_dict['url'] = request_data.findtext('url', 'unknown') + component = request_data.findtext('component') + action = request_data.findtext('action') + if component and action: + error_dict['view_name'] = '%s:%s' % (component, action) + for node in cgi_data: + key = node.get('key') + if key.startswith('HTTP') or key in whitelist_environ: + error_dict['request'][key] = node.text + elif 'query_parameters' in key: + error_dict['request']['GET'] = {} + for x in node: + error_dict['request']['GET'][x.get('key')] = x.text + elif 'request_parameters' in key: + error_dict['request']['POST'] = {} + for x in node: + error_dict['request']['POST'][x.get('key')] = x.text + elif key.endswith('cookie'): + error_dict['request']['COOKIE'] = {} + for x in node: + error_dict['request']['COOKIE'][x.get('key')] = x.text + elif key.endswith('request_id'): + error_dict['request_id'] = node.text + elif key.endswith('session'): + error_dict['request']['SESSION'] = {} + for x in node: + error_dict['request']['SESSION'][x.get('key')] = x.text + else: + if key in ['rack.session.options']: + # skip secret configs + continue + try: + if len(node): + error_dict['request'][key] = dict( + [(x.get('key'), x.text,) for x in node]) + else: + error_dict['request'][key] = node.text + except Exception as e: + log.warning('Airbrake integration exception: %s' % e) + + error_dict['request'].pop('HTTP_COOKIE', '') + + error_dict['ip'] = error_dict.pop('REMOTE_ADDR', '') + error_dict['user_agent'] = error_dict.pop('HTTP_USER_AGENT', '') + if 'request_id' not in error_dict: + error_dict['request_id'] = str(uuid.uuid4()) + if request.context.possibly_public: + # set ip for reports that come from airbrake js client + error_dict["timestamp"] = datetime.utcnow() + if request.environ.get("HTTP_X_FORWARDED_FOR"): + ip = request.environ.get("HTTP_X_FORWARDED_FOR", '') + first_ip = ip.split(',')[0] + remote_addr = first_ip.strip() + else: + remote_addr = (request.environ.get("HTTP_X_REAL_IP") or + request.environ.get('REMOTE_ADDR')) + error_dict["ip"] = remote_addr + + blacklist = ['password', 'passwd', 'pwd', 'auth_tkt', 'secret', 'csrf', + 'session', 'test'] + + lines = [] + for l in error.find('backtrace'): + lines.append({'file': l.get("file", ""), + 'line': l.get("number", ""), + 'fn': l.get("method", ""), + 'module': l.get("module", ""), + 'cline': l.get("method", ""), + 'vars': {}}) + error_dict['traceback'] = list(reversed(lines)) + # filtering is not provided by airbrake + keys_to_check = ( + error_dict['request'].get('COOKIE'), + error_dict['request'].get('COOKIES'), + error_dict['request'].get('POST'), + error_dict['request'].get('SESSION'), + ) + for source in [_f for _f in keys_to_check if _f]: + for k in source.keys(): + for bad_key in blacklist: + if bad_key in k.lower(): + source[k] = '***' + + return error_dict diff --git a/backend/src/appenlight/lib/utils/date_utils.py b/backend/src/appenlight/lib/utils/date_utils.py new file mode 100644 index 0000000..f66eece --- /dev/null +++ b/backend/src/appenlight/lib/utils/date_utils.py @@ -0,0 +1,61 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2010-2016 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# App Enlight Enterprise Edition, including its added features, Support +# services, and proprietary license terms, please see +# https://rhodecode.com/licenses/ + +from datetime import tzinfo, timedelta, datetime +from dateutil.relativedelta import relativedelta +import logging + +log = logging.getLogger(__name__) + + +def to_relativedelta(time_delta): + return relativedelta(seconds=int(time_delta.total_seconds()), + microseconds=time_delta.microseconds) + + +def convert_date(date_str, return_utcnow_if_wrong=True, + normalize_future=False): + utcnow = datetime.utcnow() + if isinstance(date_str, datetime): + # get rid of tzinfo + return date_str.replace(tzinfo=None) + if not date_str and return_utcnow_if_wrong: + return utcnow + try: + try: + if 'Z' in date_str: + date_str = date_str[:date_str.index('Z')] + if '.' in date_str: + date = datetime.strptime(date_str, '%Y-%m-%dT%H:%M:%S.%f') + else: + date = datetime.strptime(date_str, '%Y-%m-%dT%H:%M:%S') + except Exception: + # bw compat with old client + date = datetime.strptime(date_str, '%Y-%m-%d %H:%M:%S,%f') + except Exception: + if return_utcnow_if_wrong: + date = utcnow + else: + date = None + if normalize_future and date and date > (utcnow + timedelta(minutes=3)): + log.warning('time %s in future + 3 min, normalizing' % date) + return utcnow + return date diff --git a/backend/src/appenlight/lib/utils/sentry.py b/backend/src/appenlight/lib/utils/sentry.py new file mode 100644 index 0000000..fea8bc7 --- /dev/null +++ b/backend/src/appenlight/lib/utils/sentry.py @@ -0,0 +1,301 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2010-2016 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# App Enlight Enterprise Edition, including its added features, Support +# services, and proprietary license terms, please see +# https://rhodecode.com/licenses/ + +from datetime import timedelta + +from appenlight.lib.enums import LogLevelPython, ParsedSentryEventType + +EXCLUDED_LOG_VARS = [ + 'args', 'asctime', 'created', 'exc_info', 'exc_text', 'filename', + 'funcName', 'levelname', 'levelno', 'lineno', 'message', 'module', 'msecs', + 'msg', 'name', 'pathname', 'process', 'processName', 'relativeCreated', + 'thread', 'threadName'] + +EXCLUDE_SENTRY_KEYS = [ + 'csp', + 'culprit', + 'event_id', + 'exception', + 'extra', + 'level', + 'logentry', + 'logger', + 'message', + 'modules', + 'platform', + 'query', + 'release', + 'request', + 'sentry.interfaces.Csp', 'sentry.interfaces.Exception', + 'sentry.interfaces.Http', 'sentry.interfaces.Message', + 'sentry.interfaces.Query', + 'sentry.interfaces.Stacktrace', + 'sentry.interfaces.Template', 'sentry.interfaces.User', + 'sentry.interfaces.csp.Csp', + 'sentry.interfaces.exception.Exception', + 'sentry.interfaces.http.Http', + 'sentry.interfaces.message.Message', + 'sentry.interfaces.query.Query', + 'sentry.interfaces.stacktrace.Stacktrace', + 'sentry.interfaces.template.Template', + 'sentry.interfaces.user.User', 'server_name', + 'stacktrace', + 'tags', + 'template', + 'time_spent', + 'timestamp', + 'user'] + + +def get_keys(list_of_keys, json_body): + for k in list_of_keys: + if k in json_body: + return json_body[k] + + +def get_logentry(json_body): + key_names = ['logentry', + 'sentry.interfaces.message.Message', + 'sentry.interfaces.Message' + ] + logentry = get_keys(key_names, json_body) + return logentry + + +def get_exception(json_body): + parsed_exception = {} + key_names = ['exception', + 'sentry.interfaces.exception.Exception', + 'sentry.interfaces.Exception' + ] + exception = get_keys(key_names, json_body) or {} + if exception: + if isinstance(exception, dict): + exception = exception['values'][0] + else: + exception = exception[0] + + parsed_exception['type'] = exception.get('type') + parsed_exception['value'] = exception.get('value') + parsed_exception['module'] = exception.get('module') + parsed_stacktrace = get_stacktrace(exception) or {} + parsed_exception = exception or {} + return parsed_exception, parsed_stacktrace + + +def get_stacktrace(json_body): + parsed_stacktrace = [] + key_names = ['stacktrace', + 'sentry.interfaces.stacktrace.Stacktrace', + 'sentry.interfaces.Stacktrace' + ] + stacktrace = get_keys(key_names, json_body) + if stacktrace: + for frame in stacktrace['frames']: + parsed_stacktrace.append( + {"cline": frame.get('context_line', ''), + "file": frame.get('filename', ''), + "module": frame.get('module', ''), + "fn": frame.get('function', ''), + "line": frame.get('lineno', ''), + "vars": list(frame.get('vars', {}).items()) + } + ) + return parsed_stacktrace + + +def get_template(json_body): + parsed_template = {} + key_names = ['template', + 'sentry.interfaces.template.Template', + 'sentry.interfaces.Template' + ] + template = get_keys(key_names, json_body) + if template: + for frame in template['frames']: + parsed_template.append( + {"cline": frame.get('context_line', ''), + "file": frame.get('filename', ''), + "fn": '', + "line": frame.get('lineno', ''), + "vars": [] + } + ) + + return parsed_template + + +def get_request(json_body): + parsed_http = {} + key_names = ['request', + 'sentry.interfaces.http.Http', + 'sentry.interfaces.Http' + ] + http = get_keys(key_names, json_body) or {} + for k, v in http.items(): + if k == 'headers': + parsed_http['headers'] = {} + for sk, sv in http['headers'].items(): + parsed_http['headers'][sk.title()] = sv + else: + parsed_http[k.lower()] = v + return parsed_http + + +def get_user(json_body): + parsed_user = {} + key_names = ['user', + 'sentry.interfaces.user.User', + 'sentry.interfaces.User' + ] + user = get_keys(key_names, json_body) + if user: + parsed_user['id'] = user.get('id') + parsed_user['username'] = user.get('username') + parsed_user['email'] = user.get('email') + parsed_user['ip_address'] = user.get('ip_address') + + return parsed_user + + +def get_query(json_body): + query = None + key_name = ['query', + 'sentry.interfaces.query.Query', + 'sentry.interfaces.Query' + ] + query = get_keys(key_name, json_body) + return query + + +def parse_sentry_event(json_body): + request_id = json_body.get('event_id') + + # required + message = json_body.get('message') + log_timestamp = json_body.get('timestamp') + level = json_body.get('level') + if isinstance(level, int): + level = LogLevelPython.key_from_value(level) + + namespace = json_body.get('logger') + language = json_body.get('platform') + + # optional + server_name = json_body.get('server_name') + culprit = json_body.get('culprit') + release = json_body.get('release') + + tags = json_body.get('tags', {}) + if hasattr(tags, 'items'): + tags = list(tags.items()) + extra = json_body.get('extra', {}) + if hasattr(extra, 'items'): + extra = list(extra.items()) + + parsed_req = get_request(json_body) + user = get_user(json_body) + template = get_template(json_body) + query = get_query(json_body) + + # other unidentified keys found + other_keys = [(k, json_body[k]) for k in json_body.keys() + if k not in EXCLUDE_SENTRY_KEYS] + + logentry = get_logentry(json_body) + if logentry: + message = logentry['message'] + + exception, stacktrace = get_exception(json_body) + + alt_stacktrace = get_stacktrace(json_body) + event_type = None + if not exception and not stacktrace and not alt_stacktrace and not template: + event_type = ParsedSentryEventType.LOG + + event_dict = { + 'log_level': level, + 'message': message, + 'namespace': namespace, + 'request_id': request_id, + 'server': server_name, + 'date': log_timestamp, + 'tags': tags + } + event_dict['tags'].extend( + [(k, v) for k, v in extra if k not in EXCLUDED_LOG_VARS]) + + # other keys can be various object types + event_dict['tags'].extend([(k, v) for k, v in other_keys + if isinstance(v, str)]) + if culprit: + event_dict['tags'].append(('sentry_culprit', culprit)) + if language: + event_dict['tags'].append(('sentry_language', language)) + if release: + event_dict['tags'].append(('sentry_release', release)) + + if exception or stacktrace or alt_stacktrace or template: + event_type = ParsedSentryEventType.ERROR_REPORT + event_dict = { + 'client': 'sentry', + 'error': message, + 'namespace': namespace, + 'request_id': request_id, + 'server': server_name, + 'start_time': log_timestamp, + 'end_time': None, + 'tags': tags, + 'extra': extra, + 'language': language, + 'view_name': json_body.get('culprit'), + 'http_status': None, + 'username': None, + 'url': parsed_req.get('url'), + 'ip': None, + 'user_agent': None, + 'request': None, + 'slow_calls': None, + 'request_stats': None, + 'traceback': None + } + + event_dict['extra'].extend(other_keys) + if release: + event_dict['tags'].append(('sentry_release', release)) + event_dict['request'] = parsed_req + if 'headers' in parsed_req: + event_dict['user_agent'] = parsed_req['headers'].get('User-Agent') + if 'env' in parsed_req: + event_dict['ip'] = parsed_req['env'].get('REMOTE_ADDR') + ts_ms = int(json_body.get('time_spent') or 0) + if ts_ms > 0: + event_dict['end_time'] = event_dict['start_time'] + \ + timedelta(milliseconds=ts_ms) + if stacktrace or alt_stacktrace or template: + event_dict['traceback'] = stacktrace or alt_stacktrace or template + for k in list(event_dict.keys()): + if event_dict[k] is None: + del event_dict[k] + if user: + event_dict['username'] = user['username'] or user['id'] \ + or user['email'] + return event_dict, event_type