formatters: added json formatter

backend/src/appenlight/lib/logging.py

@@ -0,0 +1,69 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# AppEnlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+
+import json
+import logging
+
+ignored_keys = ['args', 'asctime', 'created', 'exc_info', 'exc_text',
+                'filename', 'funcName', 'levelname', 'levelno', 'lineno',
+                'message', 'module', 'msecs', 'msg', 'name', 'pathname',
+                'process', 'processName', 'relativeCreated', 'stack_info',
+                'thread', 'threadName']
+
+
+class JSONFormatter(logging.Formatter):
+    def format(self, record):
+        """
+        Format the specified record as text.
+
+        The record's attribute dictionary is used as the operand to a
+        string formatting operation which yields the returned string.
+        Before formatting the dictionary, a couple of preparatory steps
+        are carried out. The message attribute of the record is computed
+        using LogRecord.getMessage(). If the formatting string uses the
+        time (as determined by a call to usesTime(), formatTime() is
+        called to format the event time. If there is exception information,
+        it is formatted using formatException() and appended to the message.
+        """
+        record.message = record.getMessage()
+        log_dict = vars(record)
+        keys = [k for k in log_dict.keys() if k not in ignored_keys]
+        payload = {'message': record.message}
+        payload.update({k: log_dict[k] for k in keys})
+        record.message = json.dumps(payload, default=lambda x: str(x))
+
+        if self.usesTime():
+            record.asctime = self.formatTime(record, self.datefmt)
+        s = self.formatMessage(record)
+        if record.exc_info:
+            # Cache the traceback text to avoid converting it multiple times
+            # (it's constant anyway)
+            if not record.exc_text:
+                record.exc_text = self.formatException(record.exc_info)
+        if record.exc_text:
+            if s[-1:] != "\n":
+                s = s + "\n"
+            s = s + record.exc_text
+        if record.stack_info:
+            if s[-1:] != "\n":
+                s = s + "\n"
+            s = s + self.formatStack(record.stack_info)
+        return s

backend/src/appenlight/templates/ini/production.ini.jinja2

@@ -1,184 +1,189 b''
 [app:appenlight]
 use = egg:appenlight
 reload_templates = false
 debug_authorization = false
 debug_notfound = false
 debug_routematch = false
 debug_templates = false
 default_locale_name = en
 sqlalchemy.url = {{ appenlight_dbstring }}
 sqlalchemy.pool_size = 10
 sqlalchemy.max_overflow = 50
 sqlalchemy.echo = false
 jinja2.directories = appenlight:templates
 jinja2.filters = nl2br = appenlight.lib.jinja2_filters.nl2br
 
 #includes
 appenlight.includes =
 
 #redis
 redis.url = redis://localhost:6379/0
 redis.redlock.url = redis://localhost:6379/3
 
 #elasticsearch
 elasticsearch.nodes = http://127.0.0.1:9200
 
 #dirs
 webassets.dir = %(here)s/webassets/
 
 # encryption
 encryption_secret = {{appenlight_encryption_secret}}
 
 #authtkt
 # uncomment if you use SSL
 # authtkt.secure = true
 authtkt.secret = {{appenlight_authtkt_secret}}
 # session settings
 redis.sessions.secret = {{appenlight_redis_session_secret}}
 redis.sessions.timeout = 86400
 
 # session cookie settings
 redis.sessions.cookie_name = appenlight
 redis.sessions.cookie_max_age = 2592000
 redis.sessions.cookie_path = /
 redis.sessions.cookie_domain =
 # uncomment if you use SSL
 redis.sessions.cookie_secure = True
 redis.sessions.cookie_httponly = True
 redis.sessions.cookie_on_exception = True
 redis.sessions.prefix = appenlight:session:
 
 #cache
 cache.regions = default_term, second, short_term, long_term
 cache.type = ext:memcached
 cache.url = 127.0.0.1:11211
 cache.lock_dir = %(here)s/data/cache/lock
 cache.second.expire = 1
 cache.short_term.expire = 60
 cache.default_term.expire = 300
 
 #mailing
 mailing.app_url = https://{{appenlight_domain}}
 mailing.from_name = AppEnlight
 mailing.from_email = no-reply@{{appenlight_domain}}
 
 ###
 # Authomatic configuration
 ###
 
 authomatic.secret =
 authomatic.pr.facebook.app_id =
 authomatic.pr.facebook.secret =
 authomatic.pr.twitter.key =
 authomatic.pr.twitter.secret =
 authomatic.pr.google.key =
 authomatic.pr.google.secret =
 authomatic.pr.github.key =
 authomatic.pr.github.secret =
 authomatic.pr.github.scope  =
 authomatic.pr.bitbucket.key =
 authomatic.pr.bitbucket.secret =
 
 #ziggurat
 ziggurat_foundations.model_locations.User = appenlight.models.user:User
 ziggurat_foundations.sign_in.username_key = sign_in_user_name
 ziggurat_foundations.sign_in.password_key = sign_in_user_password
 ziggurat_foundations.sign_in.came_from_key = came_from
 
 #cometd
 cometd.server = http://127.0.0.1:8088
 cometd.secret = secret
 cometd.ws_url = wss://{{appenlight_domain}}/channelstream
 
 # for celery
 appenlight.api_key =
 appenlight.transport_config =
 appenlight.public_api_key =
 
 # celery
 celery.broker_type = redis
 celery.broker_url = redis://localhost:6379/3
 celery.concurrency = 2
 celery.timezone = UTC
 
 # tasks
 
 # how often run alerting tasks (60s default)
 tasks.notifications_reports.interval = 60
 
 [filter:paste_prefix]
 use = egg:PasteDeploy#prefix
 
 
 [filter:appenlight_client]
 use = egg:appenlight_client
 appenlight.api_key =
 
 # appenlight.transport_config = http://127.0.0.1:6543?threaded=1&timeout=5&verify=0
 # by default uses api.appenlight.com server
 appenlight.transport_config =
 appenlight.report_local_vars = true
 appenlight.report_404 = true
 appenlight.timing.dbapi2_psycopg2 = 0.3
 
 
 
 [pipeline:main]
 pipeline = paste_prefix
            appenlight_client
            appenlight
 
 
 
 [server:main]
 use = egg:gunicorn#main
 host = 0.0.0.0:6543, unix:/tmp/appenlight.sock
 workers = 2
 timeout = 90
 max_requests = 10000
 
 [server:api]
 use = egg:gunicorn#main
 host = 0.0.0.0:6553, unix:/tmp/api.appenlight.sock
 workers = 2
 max_requests = 10000
 
 
 # Begin logging configuration
 
 [loggers]
 keys = root, appenlight, sqlalchemy
 
 [handlers]
 keys = console
 
 [formatters]
-keys = generic
+keys = generic, json
 
 [logger_root]
 level = WARN
 handlers = console
 
 [logger_appenlight]
 level = WARN
 handlers =
 qualname = appenlight
 
 [logger_sqlalchemy]
 level = WARN
 handlers =
 qualname = sqlalchemy.engine
 # "level = INFO" logs SQL queries.
 # "level = DEBUG" logs SQL queries and results.
 # "level = WARN" logs neither.  (Recommended for production systems.)
 
 [handler_console]
 class = StreamHandler
 args = (sys.stderr,)
 level = NOTSET
-formatter = generic
+formatter = json
 
 [formatter_generic]
 format = %(asctime)s %(levelname)-5.5s [%(name)s][%(threadName)s] %(message)s
 
+# json string will land as "message" key of format string
+[formatter_json]
+class=appenlight.lib.logging.JSONFormatter
+format = %(asctime)s %(levelname)-5.5s [%(name)s:%(lineno)s][%(threadName)s] %(message)s
+
 # End logging configuration

development.ini

@@ -1,196 +1,201 b''
 [app:appenlight]
 use = egg:appenlight
 reload_templates = true
 debug_authorization = true
 debug_notfound = true
 debug_routematch = true
 debug_templates = true
 default_locale_name = en
 sqlalchemy.url = postgresql://test:test@localhost/appenlight
 sqlalchemy.pool_size = 10
 sqlalchemy.max_overflow = 50
 sqlalchemy.echo = false
 jinja2.directories = appenlight:templates
 jinja2.filters = nl2br = appenlight.lib.jinja2_filters.nl2br
 
 
 pyramid.includes = pyramid_debugtoolbar
 
 debugtoolbar.panels =
     pyramid_debugtoolbar.panels.versions.VersionDebugPanel
     pyramid_debugtoolbar.panels.settings.SettingsDebugPanel
     pyramid_debugtoolbar.panels.headers.HeaderDebugPanel
     pyramid_debugtoolbar.panels.renderings.RenderingsDebugPanel
     pyramid_debugtoolbar.panels.logger.LoggingPanel
     pyramid_debugtoolbar.panels.performance.PerformanceDebugPanel
     pyramid_debugtoolbar.panels.routes.RoutesDebugPanel
     pyramid_debugtoolbar.panels.sqla.SQLADebugPanel
     pyramid_debugtoolbar.panels.tweens.TweensDebugPanel
     pyramid_debugtoolbar.panels.introspection.IntrospectionDebugPanel
 
 appenlight.includes =
 
 # encryption
 encryption_secret = oEOikr_T98wTh_xLH3w8Se3kmbgAQYSM4poZvPosya0=
 
 #redis
 redis.url = redis://localhost:6379/0
 redis.redlock.url = redis://localhost:6379/3
 
 #elasticsearch
 elasticsearch.nodes = http://127.0.0.1:9200
 
 #dirs
 webassets.dir = %(here)s/webassets/
 
 #authtkt
 authtkt.secure = false
 authtkt.secret = SECRET
 # session settings
 redis.sessions.secret = SECRET
 redis.sessions.timeout = 86400
 
 # session cookie settings
 redis.sessions.cookie_name = appenlight
 redis.sessions.cookie_max_age = 2592000
 redis.sessions.cookie_path = /
 redis.sessions.cookie_domain =
 redis.sessions.cookie_secure = False
 redis.sessions.cookie_httponly = False
 redis.sessions.cookie_on_exception = True
 redis.sessions.prefix = appenlight:session:
 
 
 #cache
 cache.regions = default_term, second, short_term, long_term
 cache.type = ext:memcached
 cache.url = 127.0.0.1:11211
 cache.lock_dir = %(here)s/data/cache/lock
 cache.second.expire = 1
 cache.short_term.expire = 60
 cache.default_term.expire = 300
 
 #mailing
 mailing.app_url = http://localhost:6543
 mailing.from_name = App Enlight LOCAL
 mailing.from_email = [e-mail]
 
 
 ###
 # Authomatic configuration
 ###
 
 authomatic.secret = SECRET
 authomatic.pr.facebook.app_id =
 authomatic.pr.facebook.secret =
 authomatic.pr.twitter.key =
 authomatic.pr.twitter.secret =
 authomatic.pr.google.key =
 authomatic.pr.google.secret =
 authomatic.pr.github.key =
 authomatic.pr.github.secret =
 authomatic.pr.github.scope = repo, public_repo, user:email
 authomatic.pr.bitbucket.key =
 authomatic.pr.bitbucket.secret =
 
 #ziggurat
 ziggurat_foundations.model_locations.User = appenlight.models.user:User
 ziggurat_foundations.sign_in.username_key = sign_in_user_name
 ziggurat_foundations.sign_in.password_key = sign_in_user_password
 ziggurat_foundations.sign_in.came_from_key = came_from
 
 #cometd
 cometd.server = http://127.0.0.1:8088/
 cometd.secret = secret
 cometd.ws_url = http://127.0.0.1:8088/
 
 
 # for celery
 appenlight.api_key =
 appenlight.transport_config =
 appenlight.public_api_key =
 
 celery.broker_type = redis
 celery.broker_url = redis://localhost:6379/3
 celery.concurrency = 4
 celery.timezone = UTC
 
 
 [filter:paste_prefix]
 use = egg:PasteDeploy#prefix
 
 
 [filter:appenlight_client]
 use = egg:appenlight_client
 appenlight.api_key =
 appenlight.transport_config =
 appenlight.report_local_vars = true
 appenlight.report_404 = true
 appenlight.logging.level = DEBUG
 appenlight.timing.dbapi2_psycopg2 = 0.3
 
 
 [pipeline:main]
 pipeline =
     paste_prefix
     appenlight_client
     appenlight
 
 
 
 [server:main]
 use = egg:waitress
 host = 0.0.0.0
 port = 6543
 
 [server:main_prod]
 use = egg:gunicorn#main
 host = 0.0.0.0:6543, unix:/tmp/appenlight.sock
 workers = 6
 timeout = 90
 #max_requests = 1000
 
 
 # Begin logging configuration
 
 [loggers]
 keys = root, appenlight, sqlalchemy, elasticsearch
 
 [handlers]
 keys = console
 
 [formatters]
-keys = generic
+keys = generic, json
 
 [logger_root]
 level = INFO
 handlers = console
 
 [logger_appenlight]
 level = INFO
 handlers =
 qualname = appenlight
 
 [logger_elasticsearch]
 level = WARN
 handlers =
 qualname = elasticsearch
 
 [logger_sqlalchemy]
 level = WARN
 handlers =
 qualname = sqlalchemy.engine
 # "level = INFO" logs SQL queries.
 # "level = DEBUG" logs SQL queries and results.
 # "level = WARN" logs neither.  (Recommended for production systems.)
 
 [handler_console]
 class = StreamHandler
 args = (sys.stderr,)
 level = NOTSET
-formatter = generic
+formatter = json
 
 [formatter_generic]
 format = %(asctime)s %(levelname)-5.5s [%(name)s][%(threadName)s] %(message)s
 
+# json string will land as "message" key of format string
+[formatter_json]
+class=appenlight.lib.logging.JSONFormatter
+format = %(asctime)s %(levelname)-5.5s [%(name)s:%(lineno)s][%(threadName)s] %(message)s
+
 # End logging configuration

testing.ini

@@ -1,182 +1,187 b''
 [app:appenlight]
 use = egg:appenlight
 reload_templates = true
 debug_authorization = true
 debug_notfound = true
 debug_routematch = true
 debug_templates = true
 default_locale_name = en
 sqlalchemy.url = postgresql://test:test@localhost/appenlight_test
 sqlalchemy.pool_size = 2
 sqlalchemy.max_overflow = 5
 sqlalchemy.echo = false
 jinja2.directories = appenlight:templates
 jinja2.filters = nl2br = appenlight.lib.jinja2_filters.nl2br
 
 
 pyramid.includes = pyramid_debugtoolbar
 
 appenlight.includes =
 
 # encryption
 encryption_secret = oEOikr_T98wTh_xLH3w8Se3kmbgAQYSM4poZvPosya0=
 
 #redis
 redis.url = redis://localhost:6379/0
 redis.redlock.url = redis://localhost:6379/3
 
 #elasticsearch
 elasticsearch.nodes = http://127.0.0.1:9200
 
 #dirs
 webassets.dir = %(here)s/webassets/
 
 #authtkt
 authtkt.secure = false
 authtkt.secret = SECRET
 # session settings
 redis.sessions.secret = SECRET
 redis.sessions.timeout = 3600
 
 # session cookie settings
 redis.sessions.cookie_name = appenlight
 redis.sessions.cookie_max_age = 2592000
 redis.sessions.cookie_path = /
 redis.sessions.cookie_domain =
 redis.sessions.cookie_secure = False
 redis.sessions.cookie_httponly = False
 redis.sessions.cookie_on_exception = True
 redis.sessions.prefix = appenlight:session:
 
 
 #cache
 cache.regions = default_term, second, short_term, long_term
 cache.type = ext:memcached
 cache.url = 127.0.0.1:11211
 cache.lock_dir = %(here)s/data/cache/lock
 cache.second.expire = 1
 cache.short_term.expire = 60
 cache.default_term.expire = 300
 
 #mailing
 mailing.app_url = https://appenlight.com
 mailing.from_name = App Enlight LOCAL
 mailing.from_email = [e-mail]
 
 
 ###
 # Authomatic configuration
 ###
 
 authomatic.secret = secret
 authomatic.pr.facebook.app_id =
 authomatic.pr.facebook.secret =
 authomatic.pr.twitter.key =
 authomatic.pr.twitter.secret =
 authomatic.pr.google.key =
 authomatic.pr.google.secret =
 authomatic.pr.github.key =
 authomatic.pr.github.secret =
 authomatic.pr.github.scope = repo, public_repo, user:email
 authomatic.pr.bitbucket.key =
 authomatic.pr.bitbucket.secret =
 
 #ziggurat
 ziggurat_foundations.model_locations.User = appenlight.models.user:User
 ziggurat_foundations.sign_in.username_key = sign_in_user_name
 ziggurat_foundations.sign_in.password_key = sign_in_user_password
 ziggurat_foundations.sign_in.came_from_key = came_from
 
 #cometd
 cometd.server = http://127.0.0.1:8088/
 cometd.secret = secret
 cometd.ws_url = wss://127.0.0.1:8088/
 
 
 # for celery
 appenlight.api_key =
 appenlight.transport_config = http://127.0.0.1:6543
 
 celery.broker_type = redis
 celery.broker_url = redis://localhost:6379/4
 celery.concurrency = 4
 celery.timezone = UTC
 celery.always_eager = true
 
 [filter:paste_prefix]
 use = egg:PasteDeploy#prefix
 
 
 [filter:appenlight_client]
 use = egg:appenlight_client
 appenlight.api_key =
 appenlight.transport_config = http://127.0.0.1:6543
 appenlight.report_local_vars = true
 appenlight.report_404 = true
 appenlight.timing.dbapi2_psycopg2 = 0.3
 
 
 [pipeline:main]
 pipeline =
     paste_prefix
     appenlight_client
     appenlight
 
 
 
 [server:main]
 use = egg:waitress
 host = 0.0.0.0
 port = 6543
 
 [server:main_prod]
 use = egg:gunicorn#main
 host = 0.0.0.0:6543, unix:/tmp/appenlight.sock
 workers = 6
 timeout = 90
 #max_requests = 1000
 
 
 # Begin logging configuration
 
 [loggers]
 keys = root, appenlight, sqlalchemy, elasticsearch
 
 [handlers]
 keys = console
 
 [formatters]
-keys = generic
+keys = generic, json
 
 [logger_root]
 level = INFO
 handlers = console
 
 [logger_appenlight]
 level = INFO
 handlers =
 qualname = appenlight
 
 [logger_elasticsearch]
 level = WARN
 handlers =
 qualname = elasticsearch
 
 [logger_sqlalchemy]
 level = WARN
 handlers =
 qualname = sqlalchemy.engine
 # "level = INFO" logs SQL queries.
 # "level = DEBUG" logs SQL queries and results.
 # "level = WARN" logs neither.  (Recommended for production systems.)
 
 [handler_console]
 class = StreamHandler
 args = (sys.stderr,)
 level = NOTSET
-formatter = generic
+formatter = json
 
 [formatter_generic]
 format = %(asctime)s %(levelname)-5.5s [%(name)s][%(threadName)s] %(message)s
 
+# json string will land as "message" key of format string
+[formatter_json]
+class=appenlight.lib.logging.JSONFormatter
+format = %(asctime)s %(levelname)-5.5s [%(name)s:%(lineno)s][%(threadName)s] %(message)s
+
 # End logging configuration