application: add debug mode that switches logging to debug.

It's no longer required to reconfigure all logging. debug flag
now does it automatically.

configs/development.ini

@@ -5,6 +5,7 b''
 ################################################################################
 
 [DEFAULT]
+## Debug flag sets all loggers to debug, and enables request tracking
 debug = true
 
 ################################################################################
@@ -414,6 +415,7 b' search.location = %(here)s/data/index'
 ########################################
 ## channelstream enables persistent connections and live notification
 ## in the system. It's also used by the chat system
+
 channelstream.enabled = false
 
 ## server address for channelstream server on the backend
@@ -490,14 +492,6 b' appenlight.request_keys_blacklist ='
 ## (by default the client ignores own entries: appenlight_client.client)
 appenlight.log_namespace_blacklist =
 
-
-################################################################################
-## WARNING: *THE LINE BELOW MUST BE UNCOMMENTED ON A PRODUCTION ENVIRONMENT*  ##
-## Debug mode will enable the interactive debugging tool, allowing ANYONE to  ##
-## execute malicious code after an exception is raised.                       ##
-################################################################################
-#set debug = false
-
 # enable debug style page
 debug_style = true
 

configs/production.ini

@@ -5,7 +5,8 b''
 ################################################################################
 
 [DEFAULT]
-debug = true
+## Debug flag sets all loggers to debug, and enables request tracking
+debug = false
 
 ################################################################################
 ##                            EMAIL CONFIGURATION                             ##
@@ -389,6 +390,7 b' search.location = %(here)s/data/index'
 ########################################
 ## channelstream enables persistent connections and live notification
 ## in the system. It's also used by the chat system
+
 channelstream.enabled = false
 
 ## server address for channelstream server on the backend
@@ -466,14 +468,6 b' appenlight.request_keys_blacklist ='
 appenlight.log_namespace_blacklist =
 
 
-################################################################################
-## WARNING: *THE LINE BELOW MUST BE UNCOMMENTED ON A PRODUCTION ENVIRONMENT*  ##
-## Debug mode will enable the interactive debugging tool, allowing ANYONE to  ##
-## execute malicious code after an exception is raised.                       ##
-################################################################################
-set debug = false
-
-
 ###########################################
 ###   MAIN RHODECODE DATABASE CONFIG    ###
 ###########################################
@@ -524,6 +518,7 b' vcs.scm_app_implementation = http'
 ## Push/Pull operations hooks protocol, available options are:
 ## `http` - use http-rpc backend (default)
 vcs.hooks.protocol = http
+
 ## Host on which this instance is listening for hooks. If vcsserver is in other location
 ## this should be adjusted.
 vcs.hooks.host = 127.0.0.1

docs/admin/enable-debug.rst

@@ -3,6 +3,16 b''
 Enabling Debug Mode
 -------------------
 
+Debug Mode will enable debug logging, and request tracking middleware. Debug Mode
+enabled DEBUG log-level which allows tracking various information about authentication
+failures, LDAP connection, email etc.
+
+The request tracking will add a special
+unique ID: `| req_id:00000000-0000-0000-0000-000000000000` at the end of each log line.
+The req_id is the same for each individual requests, it means that if you want to
+track particular user logs only, and exclude other concurrent ones
+simply grep by `req_id` uuid which you'll have to find for the individual request.
+
 To enable debug mode on a |RCE| instance you need to set the debug property
 in the :file:`/home/{user}/.rccontrol/{instance-id}/rhodecode.ini` file. To
 do this, use the following steps
@@ -11,14 +21,10 b' 1. Open the file and set the ``debug`` l'
 2. Restart you instance using the ``rccontrol restart`` command,
    see the following example:
 
-You can also set the log level, the follow are the valid options;
-``debug``, ``info``, ``warning``, or ``fatal``.
-
 .. code-block:: ini
 
     [DEFAULT]
     debug = true
-    pdebug = false
 
 .. code-block:: bash
 
@@ -27,6 +33,7 b' You can also set the log level, the foll'
     Instance "enterprise-1" successfully stopped.
     Instance "enterprise-1" successfully started.
 
+
 Debug and Logging Configuration
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
@@ -47,7 +54,7 b' the ``debug`` level.'
     ### LOGGING CONFIGURATION   ####
     ################################
     [loggers]
-    keys = root, sqlalchemy, rhodecode, ssh_wrapper
+    keys = root, sqlalchemy, beaker, celery, rhodecode, ssh_wrapper
 
     [handlers]
     keys = console, console_sql, file, file_rotating
@@ -62,11 +69,16 b' the ``debug`` level.'
     level = NOTSET
     handlers = console
 
-    [logger_routes]
+    [logger_sqlalchemy]
+    level = INFO
+    handlers = console_sql
+    qualname = sqlalchemy.engine
+    propagate = 0
+
+    [logger_beaker]
     level = DEBUG
     handlers =
-    qualname = routes.middleware
-    ## "level = DEBUG" logs the route matched and routing variables.
+    qualname = beaker.container
     propagate = 1
 
     [logger_rhodecode]
@@ -75,11 +87,16 b' the ``debug`` level.'
     qualname = rhodecode
     propagate = 1
 
-    [logger_sqlalchemy]
-    level = INFO
-    handlers = console_sql
-    qualname = sqlalchemy.engine
-    propagate = 0
+    [logger_ssh_wrapper]
+    level = DEBUG
+    handlers =
+    qualname = ssh_wrapper
+    propagate = 1
+
+    [logger_celery]
+    level = DEBUG
+    handlers =
+    qualname = celery
 
     ##############
     ## HANDLERS ##
@@ -88,18 +105,18 b' the ``debug`` level.'
     [handler_console]
     class = StreamHandler
     args = (sys.stderr,)
-    level = INFO
+    level = DEBUG
     formatter = generic
 
     [handler_console_sql]
     class = StreamHandler
     args = (sys.stderr,)
-    level = WARN
+    level = INFO
     formatter = generic
 
     [handler_file]
     class = FileHandler
-    args = ('rhodecode.log', 'a',)
+    args = ('rhodecode_debug.log', 'a',)
     level = INFO
     formatter = generic
 
@@ -107,6 +124,25 b' the ``debug`` level.'
     class = logging.handlers.TimedRotatingFileHandler
     # 'D', 5 - rotate every 5days
     # you can set 'h', 'midnight'
-    args = ('rhodecode.log', 'D', 5, 10,)
+    args = ('rhodecode_debug_rotated.log', 'D', 5, 10,)
     level = INFO
     formatter = generic
+
+    ################
+    ## FORMATTERS ##
+    ################
+
+    [formatter_generic]
+    class = rhodecode.lib.logging_formatter.ExceptionAwareFormatter
+    format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
+    datefmt = %Y-%m-%d %H:%M:%S
+
+    [formatter_color_formatter]
+    class = rhodecode.lib.logging_formatter.ColorFormatter
+    format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
+    datefmt = %Y-%m-%d %H:%M:%S
+
+    [formatter_color_formatter_sql]
+    class = rhodecode.lib.logging_formatter.ColorFormatterSql
+    format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
+    datefmt = %Y-%m-%d %H:%M:%S
  No newline at end of file

rhodecode/config/middleware.py

@@ -89,6 +89,10 b' def make_pyramid_app(global_config, **se'
     # will be replaced by the value of the environment variable "NAME" in this case.
     start_time = time.time()
 
+    debug = asbool(global_config.get('debug'))
+    if debug:
+        enable_debug()
+
     environ = {'ENV_{}'.format(key): value for key, value in os.environ.items()}
 
     global_config = _substitute_values(global_config, environ)
@@ -118,6 +122,7 b' def make_pyramid_app(global_config, **se'
     total_time = time.time() - start_time
     log.info('Pyramid app `%s` created and configured in %.2fs',
              pyramid_app.func_name, total_time)
+
     return pyramid_app
 
 
@@ -432,6 +437,114 b' def sanitize_settings_and_apply_defaults'
     return settings
 
 
+def enable_debug():
+    """
+    Helper to enable debug on running instance
+    :return:
+    """
+    import tempfile
+    import textwrap
+    import logging.config
+
+    ini_template = textwrap.dedent("""
+    #####################################
+    ### DEBUG LOGGING CONFIGURATION  ####
+    #####################################
+    [loggers]
+    keys = root, sqlalchemy, beaker, celery, rhodecode, ssh_wrapper
+
+    [handlers]
+    keys = console, console_sql
+
+    [formatters]
+    keys = generic, color_formatter, color_formatter_sql
+
+    #############
+    ## LOGGERS ##
+    #############
+    [logger_root]
+    level = NOTSET
+    handlers = console
+
+    [logger_sqlalchemy]
+    level = INFO
+    handlers = console_sql
+    qualname = sqlalchemy.engine
+    propagate = 0
+
+    [logger_beaker]
+    level = DEBUG
+    handlers =
+    qualname = beaker.container
+    propagate = 1
+
+    [logger_rhodecode]
+    level = DEBUG
+    handlers =
+    qualname = rhodecode
+    propagate = 1
+
+    [logger_ssh_wrapper]
+    level = DEBUG
+    handlers =
+    qualname = ssh_wrapper
+    propagate = 1
+
+    [logger_celery]
+    level = DEBUG
+    handlers =
+    qualname = celery
+
+
+    ##############
+    ## HANDLERS ##
+    ##############
+
+    [handler_console]
+    class = StreamHandler
+    args = (sys.stderr, )
+    level = DEBUG
+    formatter = color_formatter
+
+    [handler_console_sql]
+    # "level = DEBUG" logs SQL queries and results.
+    # "level = INFO" logs SQL queries.
+    # "level = WARN" logs neither.  (Recommended for production systems.)
+    class = StreamHandler
+    args = (sys.stderr, )
+    level = WARN
+    formatter = color_formatter_sql
+
+    ################
+    ## FORMATTERS ##
+    ################
+
+    [formatter_generic]
+    class = rhodecode.lib.logging_formatter.ExceptionAwareFormatter
+    format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s | %(req_id)s
+    datefmt = %Y-%m-%d %H:%M:%S
+
+    [formatter_color_formatter]
+    class = rhodecode.lib.logging_formatter.ColorRequestTrackingFormatter
+    format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s | %(req_id)s
+    datefmt = %Y-%m-%d %H:%M:%S
+
+    [formatter_color_formatter_sql]
+    class = rhodecode.lib.logging_formatter.ColorFormatterSql
+    format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
+    datefmt = %Y-%m-%d %H:%M:%S    
+    """)
+
+    with tempfile.NamedTemporaryFile(prefix='rc_debug_logging_', suffix='.ini',
+                                     delete=False) as f:
+        log.info('Saved Temporary DEBUG config at %s', f.name)
+        f.write(ini_template)
+
+    logging.config.fileConfig(f.name)
+    log.debug('DEBUG MODE ON')
+    os.remove(f.name)
+
+
 def _sanitize_appenlight_settings(settings):
     _bool_setting(settings, 'appenlight', 'false')
 

rhodecode/public/js/rhodecode/routes.js

@@ -14,7 +14,6 b' function registerRCRoutes() {'
     // routes registration
     pyroutes.register('favicon', '/favicon.ico', []);
     pyroutes.register('robots', '/robots.txt', []);
-    pyroutes.register('auth_home', '/_admin/auth*traverse', []);
     pyroutes.register('global_integrations_new', '/_admin/integrations/new', []);
     pyroutes.register('global_integrations_home', '/_admin/integrations', []);
     pyroutes.register('global_integrations_list', '/_admin/integrations/%(integration)s', ['integration']);
@@ -30,6 +29,7 b' function registerRCRoutes() {'
     pyroutes.register('repo_integrations_list', '/%(repo_name)s/settings/integrations/%(integration)s', ['repo_name', 'integration']);
     pyroutes.register('repo_integrations_create', '/%(repo_name)s/settings/integrations/%(integration)s/new', ['repo_name', 'integration']);
     pyroutes.register('repo_integrations_edit', '/%(repo_name)s/settings/integrations/%(integration)s/%(integration_id)s', ['repo_name', 'integration', 'integration_id']);
+    pyroutes.register('auth_home', '/_admin/auth*traverse', []);
     pyroutes.register('ops_ping', '/_admin/ops/ping', []);
     pyroutes.register('ops_error_test', '/_admin/ops/error', []);
     pyroutes.register('ops_redirect_test', '/_admin/ops/redirect', []);