metrics: further adjustments.

- reduce cardinality
- added push/pull

rhodecode/config/middleware.py

@@ -1,796 +1,796 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import os
 import sys
 import logging
 import collections
 import tempfile
 import time
 
 from paste.gzipper import make_gzip_middleware
 import pyramid.events
 from pyramid.wsgi import wsgiapp
 from pyramid.authorization import ACLAuthorizationPolicy
 from pyramid.config import Configurator
 from pyramid.settings import asbool, aslist
 from pyramid.httpexceptions import (
     HTTPException, HTTPError, HTTPInternalServerError, HTTPFound, HTTPNotFound)
 from pyramid.renderers import render_to_response
 
 from rhodecode.model import meta
 from rhodecode.config import patches
 from rhodecode.config import utils as config_utils
 from rhodecode.config.environment import load_pyramid_environment
 
 import rhodecode.events
 from rhodecode.lib.middleware.vcs import VCSMiddleware
 from rhodecode.lib.request import Request
 from rhodecode.lib.vcs import VCSCommunicationError
 from rhodecode.lib.exceptions import VCSServerUnavailable
 from rhodecode.lib.middleware.appenlight import wrap_in_appenlight_if_enabled
 from rhodecode.lib.middleware.https_fixup import HttpsFixup
 from rhodecode.lib.plugins.utils import register_rhodecode_plugin
 from rhodecode.lib.utils2 import aslist as rhodecode_aslist, AttributeDict
 from rhodecode.lib.exc_tracking import store_exception
 from rhodecode.subscribers import (
     scan_repositories_if_enabled, write_js_routes_if_enabled,
     write_metadata_if_needed, write_usage_data)
 from rhodecode.lib.statsd_client import StatsdClient
 
 log = logging.getLogger(__name__)
 
 
 def is_http_error(response):
     # error which should have traceback
     return response.status_code > 499
 
 
 def should_load_all():
     """
     Returns if all application components should be loaded. In some cases it's
     desired to skip apps loading for faster shell script execution
     """
     ssh_cmd = os.environ.get('RC_CMD_SSH_WRAPPER')
     if ssh_cmd:
         return False
 
     return True
 
 
 def make_pyramid_app(global_config, **settings):
     """
     Constructs the WSGI application based on Pyramid.
 
     Specials:
 
     * The application can also be integrated like a plugin via the call to
       `includeme`. This is accompanied with the other utility functions which
       are called. Changing this should be done with great care to not break
       cases when these fragments are assembled from another place.
 
     """
 
     # Allows to use format style "{ENV_NAME}" placeholders in the configuration. It
     # will be replaced by the value of the environment variable "NAME" in this case.
     start_time = time.time()
     log.info('Pyramid app config starting')
 
     # init and bootstrap StatsdClient
     StatsdClient.setup(settings)
 
     debug = asbool(global_config.get('debug'))
     if debug:
         enable_debug()
 
     environ = {'ENV_{}'.format(key): value for key, value in os.environ.items()}
 
     global_config = _substitute_values(global_config, environ)
     settings = _substitute_values(settings, environ)
 
     sanitize_settings_and_apply_defaults(global_config, settings)
 
     config = Configurator(settings=settings)
     # Init our statsd at very start
     config.registry.statsd = StatsdClient.statsd
 
     # Apply compatibility patches
     patches.inspect_getargspec()
 
     load_pyramid_environment(global_config, settings)
 
     # Static file view comes first
     includeme_first(config)
 
     includeme(config)
 
     pyramid_app = config.make_wsgi_app()
     pyramid_app = wrap_app_in_wsgi_middlewares(pyramid_app, config)
     pyramid_app.config = config
 
     config.configure_celery(global_config['__file__'])
 
     # creating the app uses a connection - return it after we are done
     meta.Session.remove()
     statsd = StatsdClient.statsd
 
     total_time = time.time() - start_time
     log.info('Pyramid app `%s` created and configured in %.2fs',
              pyramid_app.func_name, total_time)
     if statsd:
         elapsed_time_ms = 1000.0 * total_time
         statsd.timing('rhodecode_app_bootstrap_timing', elapsed_time_ms, tags=[
             "pyramid_app:{}".format(pyramid_app.func_name)
         ])
     return pyramid_app
 
 
 def not_found_view(request):
     """
     This creates the view which should be registered as not-found-view to
     pyramid.
     """
 
     if not getattr(request, 'vcs_call', None):
         # handle like regular case with our error_handler
         return error_handler(HTTPNotFound(), request)
 
     # handle not found view as a vcs call
     settings = request.registry.settings
     ae_client = getattr(request, 'ae_client', None)
     vcs_app = VCSMiddleware(
         HTTPNotFound(), request.registry, settings,
         appenlight_client=ae_client)
 
     return wsgiapp(vcs_app)(None, request)
 
 
 def error_handler(exception, request):
     import rhodecode
     from rhodecode.lib import helpers
     from rhodecode.lib.utils2 import str2bool
 
     rhodecode_title = rhodecode.CONFIG.get('rhodecode_title') or 'RhodeCode'
 
     base_response = HTTPInternalServerError()
     # prefer original exception for the response since it may have headers set
     if isinstance(exception, HTTPException):
         base_response = exception
     elif isinstance(exception, VCSCommunicationError):
         base_response = VCSServerUnavailable()
 
     if is_http_error(base_response):
         log.exception(
             'error occurred handling this request for path: %s', request.path)
 
     statsd = request.registry.statsd
     if statsd and base_response.status_code > 499:
-        statsd.incr('rhodecode_exception')
+        statsd.incr('rhodecode_exception', tags=["code:{}".format(base_response.status_code)])
 
     error_explanation = base_response.explanation or str(base_response)
     if base_response.status_code == 404:
         error_explanation += " Optionally you don't have permission to access this page."
     c = AttributeDict()
     c.error_message = base_response.status
     c.error_explanation = error_explanation
     c.visual = AttributeDict()
 
     c.visual.rhodecode_support_url = (
         request.registry.settings.get('rhodecode_support_url') or
         request.route_url('rhodecode_support')
     )
     c.redirect_time = 0
     c.rhodecode_name = rhodecode_title
     if not c.rhodecode_name:
         c.rhodecode_name = 'Rhodecode'
 
     c.causes = []
     if is_http_error(base_response):
         c.causes.append('Server is overloaded.')
         c.causes.append('Server database connection is lost.')
         c.causes.append('Server expected unhandled error.')
 
     if hasattr(base_response, 'causes'):
         c.causes = base_response.causes
 
     c.messages = helpers.flash.pop_messages(request=request)
 
     exc_info = sys.exc_info()
     c.exception_id = id(exc_info)
     c.show_exception_id = isinstance(base_response, VCSServerUnavailable) \
                           or base_response.status_code > 499
     c.exception_id_url = request.route_url(
         'admin_settings_exception_tracker_show', exception_id=c.exception_id)
 
     if c.show_exception_id:
         store_exception(c.exception_id, exc_info)
     c.exception_debug = str2bool(rhodecode.CONFIG.get('debug'))
     c.exception_config_ini = rhodecode.CONFIG.get('__file__')
 
     response = render_to_response(
         '/errors/error_document.mako', {'c': c, 'h': helpers}, request=request,
         response=base_response)
 
     return response
 
 
 def includeme_first(config):
     # redirect automatic browser favicon.ico requests to correct place
     def favicon_redirect(context, request):
         return HTTPFound(
             request.static_path('rhodecode:public/images/favicon.ico'))
 
     config.add_view(favicon_redirect, route_name='favicon')
     config.add_route('favicon', '/favicon.ico')
 
     def robots_redirect(context, request):
         return HTTPFound(
             request.static_path('rhodecode:public/robots.txt'))
 
     config.add_view(robots_redirect, route_name='robots')
     config.add_route('robots', '/robots.txt')
 
     config.add_static_view(
         '_static/deform', 'deform:static')
     config.add_static_view(
         '_static/rhodecode', path='rhodecode:public', cache_max_age=3600 * 24)
 
 
 def includeme(config, auth_resources=None):
     from rhodecode.lib.celerylib.loader import configure_celery
     log.debug('Initializing main includeme from %s', os.path.basename(__file__))
     settings = config.registry.settings
     config.set_request_factory(Request)
 
     # plugin information
     config.registry.rhodecode_plugins = collections.OrderedDict()
 
     config.add_directive(
         'register_rhodecode_plugin', register_rhodecode_plugin)
 
     config.add_directive('configure_celery', configure_celery)
 
     if asbool(settings.get('appenlight', 'false')):
         config.include('appenlight_client.ext.pyramid_tween')
 
     load_all = should_load_all()
 
     # Includes which are required. The application would fail without them.
     config.include('pyramid_mako')
     config.include('rhodecode.lib.rc_beaker')
     config.include('rhodecode.lib.rc_cache')
     config.include('rhodecode.apps._base.navigation')
     config.include('rhodecode.apps._base.subscribers')
     config.include('rhodecode.tweens')
     config.include('rhodecode.authentication')
 
     if load_all:
         ce_auth_resources = [
             'rhodecode.authentication.plugins.auth_crowd',
             'rhodecode.authentication.plugins.auth_headers',
             'rhodecode.authentication.plugins.auth_jasig_cas',
             'rhodecode.authentication.plugins.auth_ldap',
             'rhodecode.authentication.plugins.auth_pam',
             'rhodecode.authentication.plugins.auth_rhodecode',
             'rhodecode.authentication.plugins.auth_token',
         ]
 
         # load CE authentication plugins
 
         if auth_resources:
             ce_auth_resources.extend(auth_resources)
 
         for resource in ce_auth_resources:
             config.include(resource)
 
         # Auto discover authentication plugins and include their configuration.
         if asbool(settings.get('auth_plugin.import_legacy_plugins', 'true')):
             from rhodecode.authentication import discover_legacy_plugins
             discover_legacy_plugins(config)
 
     # apps
     if load_all:
         config.include('rhodecode.api')
         config.include('rhodecode.apps._base')
         config.include('rhodecode.apps.hovercards')
         config.include('rhodecode.apps.ops')
         config.include('rhodecode.apps.channelstream')
         config.include('rhodecode.apps.file_store')
         config.include('rhodecode.apps.admin')
         config.include('rhodecode.apps.login')
         config.include('rhodecode.apps.home')
         config.include('rhodecode.apps.journal')
 
         config.include('rhodecode.apps.repository')
         config.include('rhodecode.apps.repo_group')
         config.include('rhodecode.apps.user_group')
         config.include('rhodecode.apps.search')
         config.include('rhodecode.apps.user_profile')
         config.include('rhodecode.apps.user_group_profile')
         config.include('rhodecode.apps.my_account')
         config.include('rhodecode.apps.gist')
 
         config.include('rhodecode.apps.svn_support')
         config.include('rhodecode.apps.ssh_support')
         config.include('rhodecode.apps.debug_style')
 
     if load_all:
         config.include('rhodecode.integrations')
 
     config.add_route('rhodecode_support', 'https://rhodecode.com/help/', static=True)
     config.add_translation_dirs('rhodecode:i18n/')
     settings['default_locale_name'] = settings.get('lang', 'en')
 
     # Add subscribers.
     if load_all:
         config.add_subscriber(scan_repositories_if_enabled,
                               pyramid.events.ApplicationCreated)
         config.add_subscriber(write_metadata_if_needed,
                               pyramid.events.ApplicationCreated)
         config.add_subscriber(write_usage_data,
                               pyramid.events.ApplicationCreated)
         config.add_subscriber(write_js_routes_if_enabled,
                               pyramid.events.ApplicationCreated)
 
     # request custom methods
     config.add_request_method(
         'rhodecode.lib.partial_renderer.get_partial_renderer',
         'get_partial_renderer')
 
     config.add_request_method(
         'rhodecode.lib.request_counter.get_request_counter',
         'request_count')
 
     # Set the authorization policy.
     authz_policy = ACLAuthorizationPolicy()
     config.set_authorization_policy(authz_policy)
 
     # Set the default renderer for HTML templates to mako.
     config.add_mako_renderer('.html')
 
     config.add_renderer(
         name='json_ext',
         factory='rhodecode.lib.ext_json_renderer.pyramid_ext_json')
 
     config.add_renderer(
         name='string_html',
         factory='rhodecode.lib.string_renderer.html')
 
     # include RhodeCode plugins
     includes = aslist(settings.get('rhodecode.includes', []))
     for inc in includes:
         config.include(inc)
 
     # custom not found view, if our pyramid app doesn't know how to handle
     # the request pass it to potential VCS handling ap
     config.add_notfound_view(not_found_view)
     if not settings.get('debugtoolbar.enabled', False):
         # disabled debugtoolbar handle all exceptions via the error_handlers
         config.add_view(error_handler, context=Exception)
 
     # all errors including 403/404/50X
     config.add_view(error_handler, context=HTTPError)
 
 
 def wrap_app_in_wsgi_middlewares(pyramid_app, config):
     """
     Apply outer WSGI middlewares around the application.
     """
     registry = config.registry
     settings = registry.settings
 
     # enable https redirects based on HTTP_X_URL_SCHEME set by proxy
     pyramid_app = HttpsFixup(pyramid_app, settings)
 
     pyramid_app, _ae_client = wrap_in_appenlight_if_enabled(
         pyramid_app, settings)
     registry.ae_client = _ae_client
 
     if settings['gzip_responses']:
         pyramid_app = make_gzip_middleware(
             pyramid_app, settings, compress_level=1)
 
     # this should be the outer most middleware in the wsgi stack since
     # middleware like Routes make database calls
     def pyramid_app_with_cleanup(environ, start_response):
         try:
             return pyramid_app(environ, start_response)
         finally:
             # Dispose current database session and rollback uncommitted
             # transactions.
             meta.Session.remove()
 
             # In a single threaded mode server, on non sqlite db we should have
             # '0 Current Checked out connections' at the end of a request,
             # if not, then something, somewhere is leaving a connection open
             pool = meta.Base.metadata.bind.engine.pool
             log.debug('sa pool status: %s', pool.status())
             log.debug('Request processing finalized')
 
     return pyramid_app_with_cleanup
 
 
 def sanitize_settings_and_apply_defaults(global_config, settings):
     """
     Applies settings defaults and does all type conversion.
 
     We would move all settings parsing and preparation into this place, so that
     we have only one place left which deals with this part. The remaining parts
     of the application would start to rely fully on well prepared settings.
 
     This piece would later be split up per topic to avoid a big fat monster
     function.
     """
 
     settings.setdefault('rhodecode.edition', 'Community Edition')
     settings.setdefault('rhodecode.edition_id', 'CE')
 
     if 'mako.default_filters' not in settings:
         # set custom default filters if we don't have it defined
         settings['mako.imports'] = 'from rhodecode.lib.base import h_filter'
         settings['mako.default_filters'] = 'h_filter'
 
     if 'mako.directories' not in settings:
         mako_directories = settings.setdefault('mako.directories', [
             # Base templates of the original application
             'rhodecode:templates',
         ])
         log.debug(
             "Using the following Mako template directories: %s",
             mako_directories)
 
     # NOTE(marcink): fix redis requirement for schema of connection since 3.X
     if 'beaker.session.type' in settings and settings['beaker.session.type'] == 'ext:redis':
         raw_url = settings['beaker.session.url']
         if not raw_url.startswith(('redis://', 'rediss://', 'unix://')):
             settings['beaker.session.url'] = 'redis://' + raw_url
 
     # Default includes, possible to change as a user
     pyramid_includes = settings.setdefault('pyramid.includes', [])
     log.debug(
         "Using the following pyramid.includes: %s",
         pyramid_includes)
 
     # TODO: johbo: Re-think this, usually the call to config.include
     # should allow to pass in a prefix.
     settings.setdefault('rhodecode.api.url', '/_admin/api')
     settings.setdefault('__file__', global_config.get('__file__'))
 
     # Sanitize generic settings.
     _list_setting(settings, 'default_encoding', 'UTF-8')
     _bool_setting(settings, 'is_test', 'false')
     _bool_setting(settings, 'gzip_responses', 'false')
 
     # Call split out functions that sanitize settings for each topic.
     _sanitize_appenlight_settings(settings)
     _sanitize_vcs_settings(settings)
     _sanitize_cache_settings(settings)
 
     # configure instance id
     config_utils.set_instance_id(settings)
 
     return settings
 
 
 def enable_debug():
     """
     Helper to enable debug on running instance
     :return:
     """
     import tempfile
     import textwrap
     import logging.config
 
     ini_template = textwrap.dedent("""
     #####################################
     ### DEBUG LOGGING CONFIGURATION  ####
     #####################################
     [loggers]
     keys = root, sqlalchemy, beaker, celery, rhodecode, ssh_wrapper
 
     [handlers]
     keys = console, console_sql
 
     [formatters]
     keys = generic, color_formatter, color_formatter_sql
 
     #############
     ## LOGGERS ##
     #############
     [logger_root]
     level = NOTSET
     handlers = console
 
     [logger_sqlalchemy]
     level = INFO
     handlers = console_sql
     qualname = sqlalchemy.engine
     propagate = 0
 
     [logger_beaker]
     level = DEBUG
     handlers =
     qualname = beaker.container
     propagate = 1
 
     [logger_rhodecode]
     level = DEBUG
     handlers =
     qualname = rhodecode
     propagate = 1
 
     [logger_ssh_wrapper]
     level = DEBUG
     handlers =
     qualname = ssh_wrapper
     propagate = 1
 
     [logger_celery]
     level = DEBUG
     handlers =
     qualname = celery
 
 
     ##############
     ## HANDLERS ##
     ##############
 
     [handler_console]
     class = StreamHandler
     args = (sys.stderr, )
     level = DEBUG
     formatter = color_formatter
 
     [handler_console_sql]
     # "level = DEBUG" logs SQL queries and results.
     # "level = INFO" logs SQL queries.
     # "level = WARN" logs neither.  (Recommended for production systems.)
     class = StreamHandler
     args = (sys.stderr, )
     level = WARN
     formatter = color_formatter_sql
 
     ################
     ## FORMATTERS ##
     ################
 
     [formatter_generic]
     class = rhodecode.lib.logging_formatter.ExceptionAwareFormatter
     format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s | %(req_id)s
     datefmt = %Y-%m-%d %H:%M:%S
 
     [formatter_color_formatter]
     class = rhodecode.lib.logging_formatter.ColorRequestTrackingFormatter
     format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s | %(req_id)s
     datefmt = %Y-%m-%d %H:%M:%S
 
     [formatter_color_formatter_sql]
     class = rhodecode.lib.logging_formatter.ColorFormatterSql
     format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
     datefmt = %Y-%m-%d %H:%M:%S    
     """)
 
     with tempfile.NamedTemporaryFile(prefix='rc_debug_logging_', suffix='.ini',
                                      delete=False) as f:
         log.info('Saved Temporary DEBUG config at %s', f.name)
         f.write(ini_template)
 
     logging.config.fileConfig(f.name)
     log.debug('DEBUG MODE ON')
     os.remove(f.name)
 
 
 def _sanitize_appenlight_settings(settings):
     _bool_setting(settings, 'appenlight', 'false')
 
 
 def _sanitize_vcs_settings(settings):
     """
     Applies settings defaults and does type conversion for all VCS related
     settings.
     """
     _string_setting(settings, 'vcs.svn.compatible_version', '')
     _string_setting(settings, 'vcs.hooks.protocol', 'http')
     _string_setting(settings, 'vcs.hooks.host', '127.0.0.1')
     _string_setting(settings, 'vcs.scm_app_implementation', 'http')
     _string_setting(settings, 'vcs.server', '')
     _string_setting(settings, 'vcs.server.protocol', 'http')
     _bool_setting(settings, 'startup.import_repos', 'false')
     _bool_setting(settings, 'vcs.hooks.direct_calls', 'false')
     _bool_setting(settings, 'vcs.server.enable', 'true')
     _bool_setting(settings, 'vcs.start_server', 'false')
     _list_setting(settings, 'vcs.backends', 'hg, git, svn')
     _int_setting(settings, 'vcs.connection_timeout', 3600)
 
     # Support legacy values of vcs.scm_app_implementation. Legacy
     # configurations may use 'rhodecode.lib.middleware.utils.scm_app_http', or
     # disabled since 4.13 'vcsserver.scm_app' which is now mapped to 'http'.
     scm_app_impl = settings['vcs.scm_app_implementation']
     if scm_app_impl in ['rhodecode.lib.middleware.utils.scm_app_http', 'vcsserver.scm_app']:
         settings['vcs.scm_app_implementation'] = 'http'
 
 
 def _sanitize_cache_settings(settings):
     temp_store = tempfile.gettempdir()
     default_cache_dir = os.path.join(temp_store, 'rc_cache')
 
     # save default, cache dir, and use it for all backends later.
     default_cache_dir = _string_setting(
         settings,
         'cache_dir',
         default_cache_dir, lower=False, default_when_empty=True)
 
     # ensure we have our dir created
     if not os.path.isdir(default_cache_dir):
         os.makedirs(default_cache_dir, mode=0o755)
 
     # exception store cache
     _string_setting(
         settings,
         'exception_tracker.store_path',
         temp_store, lower=False, default_when_empty=True)
     _bool_setting(
         settings,
         'exception_tracker.send_email',
         'false')
     _string_setting(
         settings,
         'exception_tracker.email_prefix',
         '[RHODECODE ERROR]', lower=False, default_when_empty=True)
 
     # cache_perms
     _string_setting(
         settings,
         'rc_cache.cache_perms.backend',
         'dogpile.cache.rc.file_namespace', lower=False)
     _int_setting(
         settings,
         'rc_cache.cache_perms.expiration_time',
         60)
     _string_setting(
         settings,
         'rc_cache.cache_perms.arguments.filename',
         os.path.join(default_cache_dir, 'rc_cache_1'), lower=False)
 
     # cache_repo
     _string_setting(
         settings,
         'rc_cache.cache_repo.backend',
         'dogpile.cache.rc.file_namespace', lower=False)
     _int_setting(
         settings,
         'rc_cache.cache_repo.expiration_time',
         60)
     _string_setting(
         settings,
         'rc_cache.cache_repo.arguments.filename',
         os.path.join(default_cache_dir, 'rc_cache_2'), lower=False)
 
     # cache_license
     _string_setting(
         settings,
         'rc_cache.cache_license.backend',
         'dogpile.cache.rc.file_namespace', lower=False)
     _int_setting(
         settings,
         'rc_cache.cache_license.expiration_time',
         5*60)
     _string_setting(
         settings,
         'rc_cache.cache_license.arguments.filename',
         os.path.join(default_cache_dir, 'rc_cache_3'), lower=False)
 
     # cache_repo_longterm memory, 96H
     _string_setting(
         settings,
         'rc_cache.cache_repo_longterm.backend',
         'dogpile.cache.rc.memory_lru', lower=False)
     _int_setting(
         settings,
         'rc_cache.cache_repo_longterm.expiration_time',
         345600)
     _int_setting(
         settings,
         'rc_cache.cache_repo_longterm.max_size',
         10000)
 
     # sql_cache_short
     _string_setting(
         settings,
         'rc_cache.sql_cache_short.backend',
         'dogpile.cache.rc.memory_lru', lower=False)
     _int_setting(
         settings,
         'rc_cache.sql_cache_short.expiration_time',
         30)
     _int_setting(
         settings,
         'rc_cache.sql_cache_short.max_size',
         10000)
 
 
 def _int_setting(settings, name, default):
     settings[name] = int(settings.get(name, default))
     return settings[name]
 
 
 def _bool_setting(settings, name, default):
     input_val = settings.get(name, default)
     if isinstance(input_val, unicode):
         input_val = input_val.encode('utf8')
     settings[name] = asbool(input_val)
     return settings[name]
 
 
 def _list_setting(settings, name, default):
     raw_value = settings.get(name, default)
 
     old_separator = ','
     if old_separator in raw_value:
         # If we get a comma separated list, pass it to our own function.
         settings[name] = rhodecode_aslist(raw_value, sep=old_separator)
     else:
         # Otherwise we assume it uses pyramids space/newline separation.
         settings[name] = aslist(raw_value)
     return settings[name]
 
 
 def _string_setting(settings, name, default, lower=True, default_when_empty=False):
     value = settings.get(name, default)
 
     if default_when_empty and not value:
         # use default value when value is empty
         value = default
 
     if lower:
         value = value.lower()
     settings[name] = value
     return settings[name]
 
 
 def _substitute_values(mapping, substitutions):
     result = {}
 
     try:
         for key, value in mapping.items():
             # initialize without substitution first
             result[key] = value
 
             # Note: Cannot use regular replacements, since they would clash
             # with the implementation of ConfigParser. Using "format" instead.
             try:
                 result[key] = value.format(**substitutions)
             except KeyError as e:
                 env_var = '{}'.format(e.args[0])
 
                 msg = 'Failed to substitute: `{key}={{{var}}}` with environment entry. ' \
                       'Make sure your environment has {var} set, or remove this ' \
                       'variable from config file'.format(key=key, var=env_var)
 
                 if env_var.startswith('ENV_'):
                     raise ValueError(msg)
                 else:
                     log.warning(msg)
 
     except ValueError as e:
         log.warning('Failed to substitute ENV variable: %s', e)
         result = mapping
 
     return result

rhodecode/lib/hooks_base.py

@@ -1,526 +1,535 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2013-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 
 """
 Set of hooks run by RhodeCode Enterprise
 """
 
 import os
 import logging
 
 import rhodecode
 from rhodecode import events
 from rhodecode.lib import helpers as h
 from rhodecode.lib import audit_logger
 from rhodecode.lib.utils2 import safe_str
 from rhodecode.lib.exceptions import (
     HTTPLockedRC, HTTPBranchProtected, UserCreationError)
 from rhodecode.model.db import Repository, User
+from rhodecode.lib.statsd_client import StatsdClient
 
 log = logging.getLogger(__name__)
 
 
 class HookResponse(object):
     def __init__(self, status, output):
         self.status = status
         self.output = output
 
     def __add__(self, other):
         other_status = getattr(other, 'status', 0)
         new_status = max(self.status, other_status)
         other_output = getattr(other, 'output', '')
         new_output = self.output + other_output
 
         return HookResponse(new_status, new_output)
 
     def __bool__(self):
         return self.status == 0
 
 
 def is_shadow_repo(extras):
     """
     Returns ``True`` if this is an action executed against a shadow repository.
     """
     return extras['is_shadow_repo']
 
 
 def _get_scm_size(alias, root_path):
 
     if not alias.startswith('.'):
         alias += '.'
 
     size_scm, size_root = 0, 0
     for path, unused_dirs, files in os.walk(safe_str(root_path)):
         if path.find(alias) != -1:
             for f in files:
                 try:
                     size_scm += os.path.getsize(os.path.join(path, f))
                 except OSError:
                     pass
         else:
             for f in files:
                 try:
                     size_root += os.path.getsize(os.path.join(path, f))
                 except OSError:
                     pass
 
     size_scm_f = h.format_byte_size_binary(size_scm)
     size_root_f = h.format_byte_size_binary(size_root)
     size_total_f = h.format_byte_size_binary(size_root + size_scm)
 
     return size_scm_f, size_root_f, size_total_f
 
 
 # actual hooks called by Mercurial internally, and GIT by our Python Hooks
 def repo_size(extras):
     """Present size of repository after push."""
     repo = Repository.get_by_repo_name(extras.repository)
     vcs_part = safe_str(u'.%s' % repo.repo_type)
     size_vcs, size_root, size_total = _get_scm_size(vcs_part,
                                                     repo.repo_full_path)
     msg = ('Repository `%s` size summary %s:%s repo:%s total:%s\n'
            % (repo.repo_name, vcs_part, size_vcs, size_root, size_total))
     return HookResponse(0, msg)
 
 
 def pre_push(extras):
     """
     Hook executed before pushing code.
 
     It bans pushing when the repository is locked.
     """
 
     user = User.get_by_username(extras.username)
     output = ''
     if extras.locked_by[0] and user.user_id != int(extras.locked_by[0]):
         locked_by = User.get(extras.locked_by[0]).username
         reason = extras.locked_by[2]
         # this exception is interpreted in git/hg middlewares and based
         # on that proper return code is server to client
         _http_ret = HTTPLockedRC(
             _locked_by_explanation(extras.repository, locked_by, reason))
         if str(_http_ret.code).startswith('2'):
             # 2xx Codes don't raise exceptions
             output = _http_ret.title
         else:
             raise _http_ret
 
     hook_response = ''
     if not is_shadow_repo(extras):
         if extras.commit_ids and extras.check_branch_perms:
 
             auth_user = user.AuthUser()
             repo = Repository.get_by_repo_name(extras.repository)
             affected_branches = []
             if repo.repo_type == 'hg':
                 for entry in extras.commit_ids:
                     if entry['type'] == 'branch':
                         is_forced = bool(entry['multiple_heads'])
                         affected_branches.append([entry['name'], is_forced])
             elif repo.repo_type == 'git':
                 for entry in extras.commit_ids:
                     if entry['type'] == 'heads':
                         is_forced = bool(entry['pruned_sha'])
                         affected_branches.append([entry['name'], is_forced])
 
             for branch_name, is_forced in affected_branches:
 
                 rule, branch_perm = auth_user.get_rule_and_branch_permission(
                     extras.repository, branch_name)
                 if not branch_perm:
                     # no branch permission found for this branch, just keep checking
                     continue
 
                 if branch_perm == 'branch.push_force':
                     continue
                 elif branch_perm == 'branch.push' and is_forced is False:
                     continue
                 elif branch_perm == 'branch.push' and is_forced is True:
                     halt_message = 'Branch `{}` changes rejected by rule {}. ' \
                                    'FORCE PUSH FORBIDDEN.'.format(branch_name, rule)
                 else:
                     halt_message = 'Branch `{}` changes rejected by rule {}.'.format(
                         branch_name, rule)
 
                 if halt_message:
                     _http_ret = HTTPBranchProtected(halt_message)
                     raise _http_ret
 
         # Propagate to external components. This is done after checking the
         # lock, for consistent behavior.
         hook_response = pre_push_extension(
             repo_store_path=Repository.base_path(), **extras)
         events.trigger(events.RepoPrePushEvent(
             repo_name=extras.repository, extras=extras))
 
     return HookResponse(0, output) + hook_response
 
 
 def pre_pull(extras):
     """
     Hook executed before pulling the code.
 
     It bans pulling when the repository is locked.
     """
 
     output = ''
     if extras.locked_by[0]:
         locked_by = User.get(extras.locked_by[0]).username
         reason = extras.locked_by[2]
         # this exception is interpreted in git/hg middlewares and based
         # on that proper return code is server to client
         _http_ret = HTTPLockedRC(
             _locked_by_explanation(extras.repository, locked_by, reason))
         if str(_http_ret.code).startswith('2'):
             # 2xx Codes don't raise exceptions
             output = _http_ret.title
         else:
             raise _http_ret
 
     # Propagate to external components. This is done after checking the
     # lock, for consistent behavior.
     hook_response = ''
     if not is_shadow_repo(extras):
         extras.hook_type = extras.hook_type or 'pre_pull'
         hook_response = pre_pull_extension(
             repo_store_path=Repository.base_path(), **extras)
         events.trigger(events.RepoPrePullEvent(
             repo_name=extras.repository, extras=extras))
 
     return HookResponse(0, output) + hook_response
 
 
 def post_pull(extras):
     """Hook executed after client pulls the code."""
 
     audit_user = audit_logger.UserWrap(
         username=extras.username,
         ip_addr=extras.ip)
     repo = audit_logger.RepoWrap(repo_name=extras.repository)
     audit_logger.store(
         'user.pull', action_data={'user_agent': extras.user_agent},
         user=audit_user, repo=repo, commit=True)
 
+    statsd = StatsdClient.statsd
+    if statsd:
+        statsd.incr('rhodecode_pull')
+
     output = ''
     # make lock is a tri state False, True, None. We only make lock on True
     if extras.make_lock is True and not is_shadow_repo(extras):
         user = User.get_by_username(extras.username)
         Repository.lock(Repository.get_by_repo_name(extras.repository),
                         user.user_id,
                         lock_reason=Repository.LOCK_PULL)
         msg = 'Made lock on repo `%s`' % (extras.repository,)
         output += msg
 
     if extras.locked_by[0]:
         locked_by = User.get(extras.locked_by[0]).username
         reason = extras.locked_by[2]
         _http_ret = HTTPLockedRC(
             _locked_by_explanation(extras.repository, locked_by, reason))
         if str(_http_ret.code).startswith('2'):
             # 2xx Codes don't raise exceptions
             output += _http_ret.title
 
     # Propagate to external components.
     hook_response = ''
     if not is_shadow_repo(extras):
         extras.hook_type = extras.hook_type or 'post_pull'
         hook_response = post_pull_extension(
             repo_store_path=Repository.base_path(), **extras)
         events.trigger(events.RepoPullEvent(
             repo_name=extras.repository, extras=extras))
 
     return HookResponse(0, output) + hook_response
 
 
 def post_push(extras):
     """Hook executed after user pushes to the repository."""
     commit_ids = extras.commit_ids
 
     # log the push call
     audit_user = audit_logger.UserWrap(
         username=extras.username, ip_addr=extras.ip)
     repo = audit_logger.RepoWrap(repo_name=extras.repository)
     audit_logger.store(
         'user.push', action_data={
             'user_agent': extras.user_agent,
             'commit_ids': commit_ids[:400]},
         user=audit_user, repo=repo, commit=True)
 
+    statsd = StatsdClient.statsd
+    if statsd:
+        statsd.incr('rhodecode_push')
+
     # Propagate to external components.
     output = ''
     # make lock is a tri state False, True, None. We only release lock on False
     if extras.make_lock is False and not is_shadow_repo(extras):
         Repository.unlock(Repository.get_by_repo_name(extras.repository))
         msg = 'Released lock on repo `{}`\n'.format(safe_str(extras.repository))
         output += msg
 
     if extras.locked_by[0]:
         locked_by = User.get(extras.locked_by[0]).username
         reason = extras.locked_by[2]
         _http_ret = HTTPLockedRC(
             _locked_by_explanation(extras.repository, locked_by, reason))
         # TODO: johbo: if not?
         if str(_http_ret.code).startswith('2'):
             # 2xx Codes don't raise exceptions
             output += _http_ret.title
 
     if extras.new_refs:
         tmpl = '{}/{}/pull-request/new?{{ref_type}}={{ref_name}}'.format(
             safe_str(extras.server_url), safe_str(extras.repository))
 
         for branch_name in extras.new_refs['branches']:
             output += 'RhodeCode: open pull request link: {}\n'.format(
                 tmpl.format(ref_type='branch', ref_name=safe_str(branch_name)))
 
         for book_name in extras.new_refs['bookmarks']:
             output += 'RhodeCode: open pull request link: {}\n'.format(
                 tmpl.format(ref_type='bookmark', ref_name=safe_str(book_name)))
 
     hook_response = ''
     if not is_shadow_repo(extras):
         hook_response = post_push_extension(
             repo_store_path=Repository.base_path(),
             **extras)
         events.trigger(events.RepoPushEvent(
             repo_name=extras.repository, pushed_commit_ids=commit_ids, extras=extras))
 
     output += 'RhodeCode: push completed\n'
     return HookResponse(0, output) + hook_response
 
 
 def _locked_by_explanation(repo_name, user_name, reason):
     message = (
         'Repository `%s` locked by user `%s`. Reason:`%s`'
         % (repo_name, user_name, reason))
     return message
 
 
 def check_allowed_create_user(user_dict, created_by, **kwargs):
     # pre create hooks
     if pre_create_user.is_active():
         hook_result = pre_create_user(created_by=created_by, **user_dict)
         allowed = hook_result.status == 0
         if not allowed:
             reason = hook_result.output
             raise UserCreationError(reason)
 
 
 class ExtensionCallback(object):
     """
     Forwards a given call to rcextensions, sanitizes keyword arguments.
 
     Does check if there is an extension active for that hook. If it is
     there, it will forward all `kwargs_keys` keyword arguments to the
     extension callback.
     """
 
     def __init__(self, hook_name, kwargs_keys):
         self._hook_name = hook_name
         self._kwargs_keys = set(kwargs_keys)
 
     def __call__(self, *args, **kwargs):
         log.debug('Calling extension callback for `%s`', self._hook_name)
         callback = self._get_callback()
         if not callback:
             log.debug('extension callback `%s` not found, skipping...', self._hook_name)
             return
 
         kwargs_to_pass = {}
         for key in self._kwargs_keys:
             try:
                 kwargs_to_pass[key] = kwargs[key]
             except KeyError:
                 log.error('Failed to fetch %s key from given kwargs. '
                           'Expected keys: %s', key, self._kwargs_keys)
                 raise
 
         # backward compat for removed api_key for old hooks. This was it works
         # with older rcextensions that require api_key present
         if self._hook_name in ['CREATE_USER_HOOK', 'DELETE_USER_HOOK']:
             kwargs_to_pass['api_key'] = '_DEPRECATED_'
         return callback(**kwargs_to_pass)
 
     def is_active(self):
         return hasattr(rhodecode.EXTENSIONS, self._hook_name)
 
     def _get_callback(self):
         return getattr(rhodecode.EXTENSIONS, self._hook_name, None)
 
 
 pre_pull_extension = ExtensionCallback(
     hook_name='PRE_PULL_HOOK',
     kwargs_keys=(
         'server_url', 'config', 'scm', 'username', 'ip', 'action',
         'repository', 'hook_type', 'user_agent', 'repo_store_path',))
 
 
 post_pull_extension = ExtensionCallback(
     hook_name='PULL_HOOK',
     kwargs_keys=(
         'server_url', 'config', 'scm', 'username', 'ip', 'action',
         'repository', 'hook_type', 'user_agent', 'repo_store_path',))
 
 
 pre_push_extension = ExtensionCallback(
     hook_name='PRE_PUSH_HOOK',
     kwargs_keys=(
         'server_url', 'config', 'scm', 'username', 'ip', 'action',
         'repository', 'repo_store_path', 'commit_ids', 'hook_type', 'user_agent',))
 
 
 post_push_extension = ExtensionCallback(
     hook_name='PUSH_HOOK',
     kwargs_keys=(
         'server_url', 'config', 'scm', 'username', 'ip', 'action',
         'repository', 'repo_store_path', 'commit_ids', 'hook_type', 'user_agent',))
 
 
 pre_create_user = ExtensionCallback(
     hook_name='PRE_CREATE_USER_HOOK',
     kwargs_keys=(
         'username', 'password', 'email', 'firstname', 'lastname', 'active',
         'admin', 'created_by'))
 
 
 create_pull_request = ExtensionCallback(
     hook_name='CREATE_PULL_REQUEST',
     kwargs_keys=(
         'server_url', 'config', 'scm', 'username', 'ip', 'action',
         'repository', 'pull_request_id', 'url', 'title', 'description',
         'status', 'created_on', 'updated_on', 'commit_ids', 'review_status',
         'mergeable', 'source', 'target', 'author', 'reviewers'))
 
 
 merge_pull_request = ExtensionCallback(
     hook_name='MERGE_PULL_REQUEST',
     kwargs_keys=(
         'server_url', 'config', 'scm', 'username', 'ip', 'action',
         'repository', 'pull_request_id', 'url', 'title', 'description',
         'status', 'created_on', 'updated_on', 'commit_ids', 'review_status',
         'mergeable', 'source', 'target', 'author', 'reviewers'))
 
 
 close_pull_request = ExtensionCallback(
     hook_name='CLOSE_PULL_REQUEST',
     kwargs_keys=(
         'server_url', 'config', 'scm', 'username', 'ip', 'action',
         'repository', 'pull_request_id', 'url', 'title', 'description',
         'status', 'created_on', 'updated_on', 'commit_ids', 'review_status',
         'mergeable', 'source', 'target', 'author', 'reviewers'))
 
 
 review_pull_request = ExtensionCallback(
     hook_name='REVIEW_PULL_REQUEST',
     kwargs_keys=(
         'server_url', 'config', 'scm', 'username', 'ip', 'action',
         'repository', 'pull_request_id', 'url', 'title', 'description',
         'status', 'created_on', 'updated_on', 'commit_ids', 'review_status',
         'mergeable', 'source', 'target', 'author', 'reviewers'))
 
 
 comment_pull_request = ExtensionCallback(
     hook_name='COMMENT_PULL_REQUEST',
     kwargs_keys=(
         'server_url', 'config', 'scm', 'username', 'ip', 'action',
         'repository', 'pull_request_id', 'url', 'title', 'description',
         'status', 'comment', 'created_on', 'updated_on', 'commit_ids', 'review_status',
         'mergeable', 'source', 'target', 'author', 'reviewers'))
 
 
 comment_edit_pull_request = ExtensionCallback(
     hook_name='COMMENT_EDIT_PULL_REQUEST',
     kwargs_keys=(
         'server_url', 'config', 'scm', 'username', 'ip', 'action',
         'repository', 'pull_request_id', 'url', 'title', 'description',
         'status', 'comment', 'created_on', 'updated_on', 'commit_ids', 'review_status',
         'mergeable', 'source', 'target', 'author', 'reviewers'))
 
 
 update_pull_request = ExtensionCallback(
     hook_name='UPDATE_PULL_REQUEST',
     kwargs_keys=(
         'server_url', 'config', 'scm', 'username', 'ip', 'action',
         'repository', 'pull_request_id', 'url', 'title', 'description',
         'status', 'created_on', 'updated_on', 'commit_ids', 'review_status',
         'mergeable', 'source', 'target', 'author', 'reviewers'))
 
 
 create_user = ExtensionCallback(
     hook_name='CREATE_USER_HOOK',
     kwargs_keys=(
         'username', 'full_name_or_username', 'full_contact', 'user_id',
         'name', 'firstname', 'short_contact', 'admin', 'lastname',
         'ip_addresses', 'extern_type', 'extern_name',
         'email', 'api_keys', 'last_login',
         'full_name', 'active', 'password', 'emails',
         'inherit_default_permissions', 'created_by', 'created_on'))
 
 
 delete_user = ExtensionCallback(
     hook_name='DELETE_USER_HOOK',
     kwargs_keys=(
         'username', 'full_name_or_username', 'full_contact', 'user_id',
         'name', 'firstname', 'short_contact', 'admin', 'lastname',
         'ip_addresses',
         'email', 'last_login',
         'full_name', 'active', 'password', 'emails',
         'inherit_default_permissions', 'deleted_by'))
 
 
 create_repository = ExtensionCallback(
     hook_name='CREATE_REPO_HOOK',
     kwargs_keys=(
         'repo_name', 'repo_type', 'description', 'private', 'created_on',
         'enable_downloads', 'repo_id', 'user_id', 'enable_statistics',
         'clone_uri', 'fork_id', 'group_id', 'created_by'))
 
 
 delete_repository = ExtensionCallback(
     hook_name='DELETE_REPO_HOOK',
     kwargs_keys=(
         'repo_name', 'repo_type', 'description', 'private', 'created_on',
         'enable_downloads', 'repo_id', 'user_id', 'enable_statistics',
         'clone_uri', 'fork_id', 'group_id', 'deleted_by', 'deleted_on'))
 
 
 comment_commit_repository = ExtensionCallback(
     hook_name='COMMENT_COMMIT_REPO_HOOK',
     kwargs_keys=(
         'repo_name', 'repo_type', 'description', 'private', 'created_on',
         'enable_downloads', 'repo_id', 'user_id', 'enable_statistics',
         'clone_uri', 'fork_id', 'group_id',
         'repository', 'created_by', 'comment', 'commit'))
 
 comment_edit_commit_repository = ExtensionCallback(
     hook_name='COMMENT_EDIT_COMMIT_REPO_HOOK',
     kwargs_keys=(
         'repo_name', 'repo_type', 'description', 'private', 'created_on',
         'enable_downloads', 'repo_id', 'user_id', 'enable_statistics',
         'clone_uri', 'fork_id', 'group_id',
         'repository', 'created_by', 'comment', 'commit'))
 
 
 create_repository_group = ExtensionCallback(
     hook_name='CREATE_REPO_GROUP_HOOK',
     kwargs_keys=(
         'group_name', 'group_parent_id', 'group_description',
         'group_id', 'user_id', 'created_by', 'created_on',
         'enable_locking'))

rhodecode/lib/middleware/request_wrapper.py

@@ -1,90 +1,90 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2016-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import time
 import logging
 
 import rhodecode
 from rhodecode.lib.auth import AuthUser
 from rhodecode.lib.base import get_ip_addr, get_access_path, get_user_agent
 from rhodecode.lib.utils2 import safe_str, get_current_rhodecode_user
 
 
 log = logging.getLogger(__name__)
 
 
 class RequestWrapperTween(object):
     def __init__(self, handler, registry):
         self.handler = handler
         self.registry = registry
 
         # one-time configuration code goes here
 
     def _get_user_info(self, request):
         user = get_current_rhodecode_user(request)
         if not user:
             user = AuthUser.repr_user(ip=get_ip_addr(request.environ))
         return user
 
     def __call__(self, request):
         start = time.time()
         log.debug('Starting request time measurement')
         try:
             response = self.handler(request)
         finally:
             count = request.request_count()
             _ver_ = rhodecode.__version__
             _path = safe_str(get_access_path(request.environ))
             _auth_user = self._get_user_info(request)
 
             total = time.time() - start
             log.info(
                 'Req[%4s] %s %s Request to %s time: %.4fs [%s], RhodeCode %s',
                 count, _auth_user, request.environ.get('REQUEST_METHOD'),
                 _path, total, get_user_agent(request. environ), _ver_
             )
 
             statsd = request.registry.statsd
             if statsd:
                 resp_code = response.status_code
                 user_id = getattr(_auth_user, 'user_id', _auth_user)
                 elapsed_time_ms = 1000.0 * total
                 statsd.timing(
                     'rhodecode_req_timing', elapsed_time_ms,
                     tags=[
-                        "path:{}".format(_path),
-                        "user:{}".format(user_id),
+                        #"path:{}".format(_path),
+                        #"user:{}".format(user_id),
                         "code:{}".format(resp_code)
                     ]
                 )
                 statsd.incr(
                     'rhodecode_req_count', tags=[
-                        "path:{}".format(_path),
-                        "user:{}".format(user_id),
+                        #"path:{}".format(_path),
+                        #"user:{}".format(user_id),
                         "code:{}".format(resp_code)
                     ])
 
         return response
 
 
 def includeme(config):
     config.add_tween(
         'rhodecode.lib.middleware.request_wrapper.RequestWrapperTween',
     )