db: move Session.remove to outer wsgi layer and also add it

to hooks daemon to avoid leaving connections open in the db pool
fixes  #4173, refs  #4166

rhodecode/config/middleware.py

@@ -1,474 +1,497 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2016  RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Pylons middleware initialization
 """
 import logging
 from collections import OrderedDict
 
 from paste.registry import RegistryManager
 from paste.gzipper import make_gzip_middleware
 from pylons.wsgiapp import PylonsApp
 from pyramid.authorization import ACLAuthorizationPolicy
 from pyramid.config import Configurator
 from pyramid.settings import asbool, aslist
 from pyramid.wsgi import wsgiapp
 from pyramid.httpexceptions import HTTPError, HTTPInternalServerError, HTTPFound
 from pyramid.events import ApplicationCreated
 import pyramid.httpexceptions as httpexceptions
 from pyramid.renderers import render_to_response
 from routes.middleware import RoutesMiddleware
 import routes.util
 
 import rhodecode
+from rhodecode.model import meta
 from rhodecode.config import patches
 from rhodecode.config.routing import STATIC_FILE_PREFIX
 from rhodecode.config.environment import (
     load_environment, load_pyramid_environment)
 from rhodecode.lib.middleware import csrf
 from rhodecode.lib.middleware.appenlight import wrap_in_appenlight_if_enabled
 from rhodecode.lib.middleware.disable_vcs import DisableVCSPagesWrapper
 from rhodecode.lib.middleware.https_fixup import HttpsFixup
 from rhodecode.lib.middleware.vcs import VCSMiddleware
 from rhodecode.lib.plugins.utils import register_rhodecode_plugin
 from rhodecode.lib.utils2 import aslist as rhodecode_aslist
 from rhodecode.subscribers import scan_repositories_if_enabled
 
 
 log = logging.getLogger(__name__)
 
 
 # this is used to avoid avoid the route lookup overhead in routesmiddleware
 # for certain routes which won't go to pylons to - eg. static files, debugger
 # it is only needed for the pylons migration and can be removed once complete
 class SkippableRoutesMiddleware(RoutesMiddleware):
     """ Routes middleware that allows you to skip prefixes """
 
     def __init__(self, *args, **kw):
         self.skip_prefixes = kw.pop('skip_prefixes', [])
         super(SkippableRoutesMiddleware, self).__init__(*args, **kw)
 
     def __call__(self, environ, start_response):
         for prefix in self.skip_prefixes:
             if environ['PATH_INFO'].startswith(prefix):
                 # added to avoid the case when a missing /_static route falls
                 # through to pylons and causes an exception as pylons is
                 # expecting wsgiorg.routingargs to be set in the environ
                 # by RoutesMiddleware.
                 if 'wsgiorg.routing_args' not in environ:
                     environ['wsgiorg.routing_args'] = (None, {})
                 return self.app(environ, start_response)
 
         return super(SkippableRoutesMiddleware, self).__call__(
             environ, start_response)
 
 
 def make_app(global_conf, static_files=True, **app_conf):
     """Create a Pylons WSGI application and return it
 
     ``global_conf``
         The inherited configuration for this application. Normally from
         the [DEFAULT] section of the Paste ini file.
 
     ``app_conf``
         The application's local configuration. Normally specified in
         the [app:<name>] section of the Paste ini file (where <name>
         defaults to main).
 
     """
     # Apply compatibility patches
     patches.kombu_1_5_1_python_2_7_11()
     patches.inspect_getargspec()
 
     # Configure the Pylons environment
     config = load_environment(global_conf, app_conf)
 
     # The Pylons WSGI app
     app = PylonsApp(config=config)
     if rhodecode.is_test:
         app = csrf.CSRFDetector(app)
 
     expected_origin = config.get('expected_origin')
     if expected_origin:
         # The API can be accessed from other Origins.
         app = csrf.OriginChecker(app, expected_origin,
                                  skip_urls=[routes.util.url_for('api')])
 
     # Establish the Registry for this application
     app = RegistryManager(app)
 
     app.config = config
 
     return app
 
 
 def make_pyramid_app(global_config, **settings):
     """
     Constructs the WSGI application based on Pyramid and wraps the Pylons based
     application.
 
     Specials:
 
     * We migrate from Pylons to Pyramid. While doing this, we keep both
       frameworks functional. This involves moving some WSGI middlewares around
       and providing access to some data internals, so that the old code is
       still functional.
 
     * The application can also be integrated like a plugin via the call to
       `includeme`. This is accompanied with the other utility functions which
       are called. Changing this should be done with great care to not break
       cases when these fragments are assembled from another place.
 
     """
     # The edition string should be available in pylons too, so we add it here
     # before copying the settings.
     settings.setdefault('rhodecode.edition', 'Community Edition')
 
     # As long as our Pylons application does expect "unprepared" settings, make
     # sure that we keep an unmodified copy. This avoids unintentional change of
     # behavior in the old application.
     settings_pylons = settings.copy()
 
     sanitize_settings_and_apply_defaults(settings)
     config = Configurator(settings=settings)
     add_pylons_compat_data(config.registry, global_config, settings_pylons)
 
     load_pyramid_environment(global_config, settings)
 
     includeme_first(config)
     includeme(config)
     pyramid_app = config.make_wsgi_app()
     pyramid_app = wrap_app_in_wsgi_middlewares(pyramid_app, config)
     pyramid_app.config = config
+
+    # creating the app uses a connection - return it after we are done
+    meta.Session.remove()
+
     return pyramid_app
 
 
 def make_not_found_view(config):
     """
     This creates the view which should be registered as not-found-view to
     pyramid. Basically it contains of the old pylons app, converted to a view.
     Additionally it is wrapped by some other middlewares.
     """
     settings = config.registry.settings
     vcs_server_enabled = settings['vcs.server.enable']
 
     # Make pylons app from unprepared settings.
     pylons_app = make_app(
         config.registry._pylons_compat_global_config,
         **config.registry._pylons_compat_settings)
     config.registry._pylons_compat_config = pylons_app.config
 
     # Appenlight monitoring.
     pylons_app, appenlight_client = wrap_in_appenlight_if_enabled(
         pylons_app, settings)
 
     # The VCSMiddleware shall operate like a fallback if pyramid doesn't find
     # a view to handle the request. Therefore we wrap it around the pylons app.
     if vcs_server_enabled:
         pylons_app = VCSMiddleware(
             pylons_app, settings, appenlight_client, registry=config.registry)
 
     pylons_app_as_view = wsgiapp(pylons_app)
 
     # Protect from VCS Server error related pages when server is not available
     if not vcs_server_enabled:
         pylons_app_as_view = DisableVCSPagesWrapper(pylons_app_as_view)
 
     def pylons_app_with_error_handler(context, request):
         """
         Handle exceptions from rc pylons app:
 
         - old webob type exceptions get converted to pyramid exceptions
         - pyramid exceptions are passed to the error handler view
         """
         def is_vcs_response(response):
             return 'X-RhodeCode-Backend' in response.headers
 
         def is_http_error(response):
             # webob type error responses
             return (400 <= response.status_int <= 599)
 
         def is_error_handling_needed(response):
             return is_http_error(response) and not is_vcs_response(response)
 
         try:
             response = pylons_app_as_view(context, request)
             if is_error_handling_needed(response):
                 response = webob_to_pyramid_http_response(response)
                 return error_handler(response, request)
         except HTTPError as e:  # pyramid type exceptions
             return error_handler(e, request)
         except Exception:
             if settings.get('debugtoolbar.enabled', False):
                 raise
             return error_handler(HTTPInternalServerError(), request)
         return response
 
     return pylons_app_with_error_handler
 
 
 def add_pylons_compat_data(registry, global_config, settings):
     """
     Attach data to the registry to support the Pylons integration.
     """
     registry._pylons_compat_global_config = global_config
     registry._pylons_compat_settings = settings
 
 
 def webob_to_pyramid_http_response(webob_response):
     ResponseClass = httpexceptions.status_map[webob_response.status_int]
     pyramid_response = ResponseClass(webob_response.status)
     pyramid_response.status = webob_response.status
     pyramid_response.headers.update(webob_response.headers)
     if pyramid_response.headers['content-type'] == 'text/html':
         pyramid_response.headers['content-type'] = 'text/html; charset=UTF-8'
     return pyramid_response
 
 
 def error_handler(exception, request):
     # TODO: dan: replace the old pylons error controller with this
     from rhodecode.model.settings import SettingsModel
     from rhodecode.lib.utils2 import AttributeDict
 
     try:
         rc_config = SettingsModel().get_all_settings()
     except Exception:
         log.exception('failed to fetch settings')
         rc_config = {}
 
     base_response = HTTPInternalServerError()
     # prefer original exception for the response since it may have headers set
     if isinstance(exception, HTTPError):
         base_response = exception
 
     c = AttributeDict()
     c.error_message = base_response.status
     c.error_explanation = base_response.explanation or str(base_response)
     c.visual = AttributeDict()
 
     c.visual.rhodecode_support_url = (
         request.registry.settings.get('rhodecode_support_url') or
         request.route_url('rhodecode_support')
     )
     c.redirect_time = 0
     c.rhodecode_name = rc_config.get('rhodecode_title', '')
     if not c.rhodecode_name:
         c.rhodecode_name = 'Rhodecode'
 
     response = render_to_response(
         '/errors/error_document.html', {'c': c}, request=request,
         response=base_response)
 
     return response
 
 
 def includeme(config):
     settings = config.registry.settings
 
     # plugin information
     config.registry.rhodecode_plugins = OrderedDict()
 
     config.add_directive(
         'register_rhodecode_plugin', register_rhodecode_plugin)
 
     if asbool(settings.get('appenlight', 'false')):
         config.include('appenlight_client.ext.pyramid_tween')
 
     # Includes which are required. The application would fail without them.
     config.include('pyramid_mako')
     config.include('pyramid_beaker')
     config.include('rhodecode.channelstream')
     config.include('rhodecode.admin')
     config.include('rhodecode.authentication')
     config.include('rhodecode.integrations')
     config.include('rhodecode.login')
     config.include('rhodecode.tweens')
     config.include('rhodecode.api')
     config.include('rhodecode.svn_support')
     config.add_route(
         'rhodecode_support', 'https://rhodecode.com/help/', static=True)
 
     # Add subscribers.
     config.add_subscriber(scan_repositories_if_enabled, ApplicationCreated)
 
     # Set the authorization policy.
     authz_policy = ACLAuthorizationPolicy()
     config.set_authorization_policy(authz_policy)
 
     # Set the default renderer for HTML templates to mako.
     config.add_mako_renderer('.html')
 
     # include RhodeCode plugins
     includes = aslist(settings.get('rhodecode.includes', []))
     for inc in includes:
         config.include(inc)
 
     # This is the glue which allows us to migrate in chunks. By registering the
     # pylons based application as the "Not Found" view in Pyramid, we will
     # fallback to the old application each time the new one does not yet know
     # how to handle a request.
     config.add_notfound_view(make_not_found_view(config))
 
     if not settings.get('debugtoolbar.enabled', False):
         # if no toolbar, then any exception gets caught and rendered
         config.add_view(error_handler, context=Exception)
 
     config.add_view(error_handler, context=HTTPError)
 
 
 def includeme_first(config):
     # redirect automatic browser favicon.ico requests to correct place
     def favicon_redirect(context, request):
         return HTTPFound(
             request.static_path('rhodecode:public/images/favicon.ico'))
 
     config.add_view(favicon_redirect, route_name='favicon')
     config.add_route('favicon', '/favicon.ico')
 
     config.add_static_view(
         '_static/deform', 'deform:static')
     config.add_static_view(
         '_static/rhodecode', path='rhodecode:public', cache_max_age=3600 * 24)
 
 
 def wrap_app_in_wsgi_middlewares(pyramid_app, config):
     """
     Apply outer WSGI middlewares around the application.
 
     Part of this has been moved up from the Pylons layer, so that the
     data is also available if old Pylons code is hit through an already ported
     view.
     """
     settings = config.registry.settings
 
     # enable https redirects based on HTTP_X_URL_SCHEME set by proxy
     pyramid_app = HttpsFixup(pyramid_app, settings)
 
     # Add RoutesMiddleware to support the pylons compatibility tween during
     # migration to pyramid.
     pyramid_app = SkippableRoutesMiddleware(
         pyramid_app, config.registry._pylons_compat_config['routes.map'],
         skip_prefixes=(STATIC_FILE_PREFIX, '/_debug_toolbar'))
 
     pyramid_app, _ = wrap_in_appenlight_if_enabled(pyramid_app, settings)
 
     if settings['gzip_responses']:
         pyramid_app = make_gzip_middleware(
             pyramid_app, settings, compress_level=1)
 
-    return pyramid_app
+
+    # this should be the outer most middleware in the wsgi stack since
+    # middleware like Routes make database calls
+    def pyramid_app_with_cleanup(environ, start_response):
+        try:
+            return pyramid_app(environ, start_response)
+        finally:
+            # Dispose current database session and rollback uncommitted
+            # transactions.
+            meta.Session.remove()
+
+            # In a single threaded mode server, on non sqlite db we should have
+            # '0 Current Checked out connections' at the end of a request,
+            # if not, then something, somewhere is leaving a connection open
+            pool = meta.Base.metadata.bind.engine.pool
+            log.debug('sa pool status: %s', pool.status())
+
+
+    return pyramid_app_with_cleanup
 
 
 def sanitize_settings_and_apply_defaults(settings):
     """
     Applies settings defaults and does all type conversion.
 
     We would move all settings parsing and preparation into this place, so that
     we have only one place left which deals with this part. The remaining parts
     of the application would start to rely fully on well prepared settings.
 
     This piece would later be split up per topic to avoid a big fat monster
     function.
     """
 
     # Pyramid's mako renderer has to search in the templates folder so that the
     # old templates still work. Ported and new templates are expected to use
     # real asset specifications for the includes.
     mako_directories = settings.setdefault('mako.directories', [
         # Base templates of the original Pylons application
         'rhodecode:templates',
     ])
     log.debug(
         "Using the following Mako template directories: %s",
         mako_directories)
 
     # Default includes, possible to change as a user
     pyramid_includes = settings.setdefault('pyramid.includes', [
         'rhodecode.lib.middleware.request_wrapper',
     ])
     log.debug(
         "Using the following pyramid.includes: %s",
         pyramid_includes)
 
     # TODO: johbo: Re-think this, usually the call to config.include
     # should allow to pass in a prefix.
     settings.setdefault('rhodecode.api.url', '/_admin/api')
 
     # Sanitize generic settings.
     _list_setting(settings, 'default_encoding', 'UTF-8')
     _bool_setting(settings, 'is_test', 'false')
     _bool_setting(settings, 'gzip_responses', 'false')
 
     # Call split out functions that sanitize settings for each topic.
     _sanitize_appenlight_settings(settings)
     _sanitize_vcs_settings(settings)
 
     return settings
 
 
 def _sanitize_appenlight_settings(settings):
     _bool_setting(settings, 'appenlight', 'false')
 
 
 def _sanitize_vcs_settings(settings):
     """
     Applies settings defaults and does type conversion for all VCS related
     settings.
     """
     _string_setting(settings, 'vcs.svn.compatible_version', '')
     _string_setting(settings, 'git_rev_filter', '--all')
     _string_setting(settings, 'vcs.hooks.protocol', 'pyro4')
     _string_setting(settings, 'vcs.server', '')
     _string_setting(settings, 'vcs.server.log_level', 'debug')
     _string_setting(settings, 'vcs.server.protocol', 'pyro4')
     _bool_setting(settings, 'startup.import_repos', 'false')
     _bool_setting(settings, 'vcs.hooks.direct_calls', 'false')
     _bool_setting(settings, 'vcs.server.enable', 'true')
     _bool_setting(settings, 'vcs.start_server', 'false')
     _list_setting(settings, 'vcs.backends', 'hg, git, svn')
     _int_setting(settings, 'vcs.connection_timeout', 3600)
 
 
 def _int_setting(settings, name, default):
     settings[name] = int(settings.get(name, default))
 
 
 def _bool_setting(settings, name, default):
     input = settings.get(name, default)
     if isinstance(input, unicode):
         input = input.encode('utf8')
     settings[name] = asbool(input)
 
 
 def _list_setting(settings, name, default):
     raw_value = settings.get(name, default)
 
     old_separator = ','
     if old_separator in raw_value:
         # If we get a comma separated list, pass it to our own function.
         settings[name] = rhodecode_aslist(raw_value, sep=old_separator)
     else:
         # Otherwise we assume it uses pyramids space/newline separation.
         settings[name] = aslist(raw_value)
 
 
 def _string_setting(settings, name, default):
     settings[name] = settings.get(name, default).lower()

rhodecode/lib/hooks_daemon.py

@@ -1,274 +1,278 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2016  RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import json
 import logging
 import urlparse
 import threading
 from BaseHTTPServer import BaseHTTPRequestHandler
 from SocketServer import TCPServer
 from routes.util import URLGenerator
 
 import Pyro4
 import pylons
 import rhodecode
 
+from rhodecode.model import meta
 from rhodecode.lib import hooks_base
 from rhodecode.lib.utils2 import (
     AttributeDict, safe_str, get_routes_generator_for_server_url)
 
 
 log = logging.getLogger(__name__)
 
 
 class HooksHttpHandler(BaseHTTPRequestHandler):
     def do_POST(self):
         method, extras = self._read_request()
         try:
             result = self._call_hook(method, extras)
         except Exception as e:
             result = {
                 'exception': e.__class__.__name__,
                 'exception_args': e.args
             }
         self._write_response(result)
 
     def _read_request(self):
         length = int(self.headers['Content-Length'])
         body = self.rfile.read(length).decode('utf-8')
         data = json.loads(body)
         return data['method'], data['extras']
 
     def _write_response(self, result):
         self.send_response(200)
         self.send_header("Content-type", "text/json")
         self.end_headers()
         self.wfile.write(json.dumps(result))
 
     def _call_hook(self, method, extras):
         hooks = Hooks()
-        result = getattr(hooks, method)(extras)
+        try:
+            result = getattr(hooks, method)(extras)
+        finally:
+            meta.Session.remove()
         return result
 
     def log_message(self, format, *args):
         """
         This is an overriden method of BaseHTTPRequestHandler which logs using
         logging library instead of writing directly to stderr.
         """
 
         message = format % args
 
         # TODO: mikhail: add different log levels support
         log.debug(
             "%s - - [%s] %s", self.client_address[0],
             self.log_date_time_string(), message)
 
 
 class DummyHooksCallbackDaemon(object):
     def __init__(self):
         self.hooks_module = Hooks.__module__
 
     def __enter__(self):
         log.debug('Running dummy hooks callback daemon')
         return self
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         log.debug('Exiting dummy hooks callback daemon')
 
 
 class ThreadedHookCallbackDaemon(object):
 
     _callback_thread = None
     _daemon = None
     _done = False
 
     def __init__(self):
         self._prepare()
 
     def __enter__(self):
         self._run()
         return self
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         self._stop()
 
     def _prepare(self):
         raise NotImplementedError()
 
     def _run(self):
         raise NotImplementedError()
 
     def _stop(self):
         raise NotImplementedError()
 
 
 class Pyro4HooksCallbackDaemon(ThreadedHookCallbackDaemon):
     """
     Context manager which will run a callback daemon in a background thread.
     """
 
     hooks_uri = None
 
     def _prepare(self):
         log.debug("Preparing callback daemon and registering hook object")
         self._daemon = Pyro4.Daemon()
         hooks_interface = Hooks()
         self.hooks_uri = str(self._daemon.register(hooks_interface))
         log.debug("Hooks uri is: %s", self.hooks_uri)
 
     def _run(self):
         log.debug("Running event loop of callback daemon in background thread")
         callback_thread = threading.Thread(
             target=self._daemon.requestLoop,
             kwargs={'loopCondition': lambda: not self._done})
         callback_thread.daemon = True
         callback_thread.start()
         self._callback_thread = callback_thread
 
     def _stop(self):
         log.debug("Waiting for background thread to finish.")
         self._done = True
         self._callback_thread.join()
         self._daemon.close()
         self._daemon = None
         self._callback_thread = None
 
 
 class HttpHooksCallbackDaemon(ThreadedHookCallbackDaemon):
     """
     Context manager which will run a callback daemon in a background thread.
     """
 
     hooks_uri = None
 
     IP_ADDRESS = '127.0.0.1'
 
     # From Python docs: Polling reduces our responsiveness to a shutdown
     # request and wastes cpu at all other times.
     POLL_INTERVAL = 0.1
 
     def _prepare(self):
         log.debug("Preparing callback daemon and registering hook object")
 
         self._done = False
         self._daemon = TCPServer((self.IP_ADDRESS, 0), HooksHttpHandler)
         _, port = self._daemon.server_address
         self.hooks_uri = '{}:{}'.format(self.IP_ADDRESS, port)
 
         log.debug("Hooks uri is: %s", self.hooks_uri)
 
     def _run(self):
         log.debug("Running event loop of callback daemon in background thread")
         callback_thread = threading.Thread(
             target=self._daemon.serve_forever,
             kwargs={'poll_interval': self.POLL_INTERVAL})
         callback_thread.daemon = True
         callback_thread.start()
         self._callback_thread = callback_thread
 
     def _stop(self):
         log.debug("Waiting for background thread to finish.")
         self._daemon.shutdown()
         self._callback_thread.join()
         self._daemon = None
         self._callback_thread = None
 
 
 def prepare_callback_daemon(extras, protocol=None, use_direct_calls=False):
     callback_daemon = None
     protocol = protocol.lower() if protocol else None
 
     if use_direct_calls:
         callback_daemon = DummyHooksCallbackDaemon()
         extras['hooks_module'] = callback_daemon.hooks_module
     else:
         if protocol == 'pyro4':
             callback_daemon = Pyro4HooksCallbackDaemon()
         elif protocol == 'http':
             callback_daemon = HttpHooksCallbackDaemon()
         else:
             log.error('Unsupported callback daemon protocol "%s"', protocol)
             raise Exception('Unsupported callback daemon protocol.')
 
         extras['hooks_uri'] = callback_daemon.hooks_uri
         extras['hooks_protocol'] = protocol
 
     return callback_daemon, extras
 
 
 class Hooks(object):
     """
     Exposes the hooks for remote call backs
     """
 
     @Pyro4.callback
     def repo_size(self, extras):
         log.debug("Called repo_size of Hooks object")
         return self._call_hook(hooks_base.repo_size, extras)
 
     @Pyro4.callback
     def pre_pull(self, extras):
         log.debug("Called pre_pull of Hooks object")
         return self._call_hook(hooks_base.pre_pull, extras)
 
     @Pyro4.callback
     def post_pull(self, extras):
         log.debug("Called post_pull of Hooks object")
         return self._call_hook(hooks_base.post_pull, extras)
 
     @Pyro4.callback
     def pre_push(self, extras):
         log.debug("Called pre_push of Hooks object")
         return self._call_hook(hooks_base.pre_push, extras)
 
     @Pyro4.callback
     def post_push(self, extras):
         log.debug("Called post_push of Hooks object")
         return self._call_hook(hooks_base.post_push, extras)
 
     def _call_hook(self, hook, extras):
         extras = AttributeDict(extras)
         pylons_router = get_routes_generator_for_server_url(extras.server_url)
         pylons.url._push_object(pylons_router)
 
         try:
             result = hook(extras)
         except Exception as error:
             log.exception('Exception when handling hook %s', hook)
             error_args = error.args
             return {
                 'status': 128,
                 'output': '',
                 'exception': type(error).__name__,
                 'exception_args': error_args,
             }
         finally:
             pylons.url._pop_object()
 
         return {
             'status': result.status,
             'output': result.output,
         }
 
     def __enter__(self):
         return self
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         pass

rhodecode/lib/middleware/simplevcs.py

@@ -1,445 +1,448 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2014-2016  RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 SimpleVCS middleware for handling protocol request (push/clone etc.)
 It's implemented with basic auth function
 """
 
 import os
 import logging
 import importlib
 from functools import wraps
 
 from paste.httpheaders import REMOTE_USER, AUTH_TYPE
 from webob.exc import (
     HTTPNotFound, HTTPForbidden, HTTPNotAcceptable, HTTPInternalServerError)
 
 import rhodecode
 from rhodecode.authentication.base import authenticate, VCS_TYPE
 from rhodecode.lib.auth import AuthUser, HasPermissionAnyMiddleware
 from rhodecode.lib.base import BasicAuth, get_ip_addr, vcs_operation_context
 from rhodecode.lib.exceptions import (
     HTTPLockedRC, HTTPRequirementError, UserCreationError,
     NotAllowedToCreateUserError)
 from rhodecode.lib.hooks_daemon import prepare_callback_daemon
 from rhodecode.lib.middleware import appenlight
 from rhodecode.lib.middleware.utils import scm_app
 from rhodecode.lib.utils import (
     is_valid_repo, get_rhodecode_realm, get_rhodecode_base_path)
 from rhodecode.lib.utils2 import safe_str, fix_PATH, str2bool
 from rhodecode.lib.vcs.conf import settings as vcs_settings
 from rhodecode.model import meta
 from rhodecode.model.db import User, Repository
 from rhodecode.model.scm import ScmModel
 from rhodecode.model.settings import SettingsModel
 
 log = logging.getLogger(__name__)
 
 
 def initialize_generator(factory):
     """
     Initializes the returned generator by draining its first element.
 
     This can be used to give a generator an initializer, which is the code
     up to the first yield statement. This decorator enforces that the first
     produced element has the value ``"__init__"`` to make its special
     purpose very explicit in the using code.
     """
 
     @wraps(factory)
     def wrapper(*args, **kwargs):
         gen = factory(*args, **kwargs)
         try:
             init = gen.next()
         except StopIteration:
             raise ValueError('Generator must yield at least one element.')
         if init != "__init__":
             raise ValueError('First yielded element must be "__init__".')
         return gen
     return wrapper
 
 
 class SimpleVCS(object):
     """Common functionality for SCM HTTP handlers."""
 
     SCM = 'unknown'
 
     def __init__(self, application, config, registry):
         self.registry = registry
         self.application = application
         self.config = config
         # base path of repo locations
         self.basepath = get_rhodecode_base_path()
         # authenticate this VCS request using authfunc
         auth_ret_code_detection = \
             str2bool(self.config.get('auth_ret_code_detection', False))
         self.authenticate = BasicAuth(
             '', authenticate, registry, config.get('auth_ret_code'),
             auth_ret_code_detection)
         self.ip_addr = '0.0.0.0'
 
     @property
     def scm_app(self):
         custom_implementation = self.config.get('vcs.scm_app_implementation')
         if custom_implementation and custom_implementation != 'pyro4':
             log.info(
                 "Using custom implementation of scm_app: %s",
                 custom_implementation)
             scm_app_impl = importlib.import_module(custom_implementation)
         else:
             scm_app_impl = scm_app
         return scm_app_impl
 
     def _get_by_id(self, repo_name):
         """
         Gets a special pattern _<ID> from clone url and tries to replace it
         with a repository_name for support of _<ID> non changable urls
 
         :param repo_name:
         """
 
         data = repo_name.split('/')
         if len(data) >= 2:
             from rhodecode.model.repo import RepoModel
             by_id_match = RepoModel().get_repo_by_id(repo_name)
             if by_id_match:
                 data[1] = by_id_match.repo_name
 
         return safe_str('/'.join(data))
 
     def _invalidate_cache(self, repo_name):
         """
         Set's cache for this repository for invalidation on next access
 
         :param repo_name: full repo name, also a cache key
         """
         ScmModel().mark_for_invalidation(repo_name)
 
     def is_valid_and_existing_repo(self, repo_name, base_path, scm_type):
         db_repo = Repository.get_by_repo_name(repo_name)
         if not db_repo:
             log.debug('Repository `%s` not found inside the database.',
                       repo_name)
             return False
 
         if db_repo.repo_type != scm_type:
             log.warning(
                 'Repository `%s` have incorrect scm_type, expected %s got %s',
                 repo_name, db_repo.repo_type, scm_type)
             return False
 
         return is_valid_repo(repo_name, base_path, expect_scm=scm_type)
 
     def valid_and_active_user(self, user):
         """
         Checks if that user is not empty, and if it's actually object it checks
         if he's active.
 
         :param user: user object or None
         :return: boolean
         """
         if user is None:
             return False
 
         elif user.active:
             return True
 
         return False
 
     def _check_permission(self, action, user, repo_name, ip_addr=None):
         """
         Checks permissions using action (push/pull) user and repository
         name
 
         :param action: push or pull action
         :param user: user instance
         :param repo_name: repository name
         """
         # check IP
         inherit = user.inherit_default_permissions
         ip_allowed = AuthUser.check_ip_allowed(user.user_id, ip_addr,
                                                inherit_from_default=inherit)
         if ip_allowed:
             log.info('Access for IP:%s allowed', ip_addr)
         else:
             return False
 
         if action == 'push':
             if not HasPermissionAnyMiddleware('repository.write',
                                               'repository.admin')(user,
                                                                   repo_name):
                 return False
 
         else:
             # any other action need at least read permission
             if not HasPermissionAnyMiddleware('repository.read',
                                               'repository.write',
                                               'repository.admin')(user,
                                                                   repo_name):
                 return False
 
         return True
 
     def _check_ssl(self, environ, start_response):
         """
         Checks the SSL check flag and returns False if SSL is not present
         and required True otherwise
         """
         org_proto = environ['wsgi._org_proto']
         # check if we have SSL required  ! if not it's a bad request !
         require_ssl = str2bool(
             SettingsModel().get_ui_by_key('push_ssl').ui_value)
         if require_ssl and org_proto == 'http':
             log.debug('proto is %s and SSL is required BAD REQUEST !',
                       org_proto)
             return False
         return True
 
     def __call__(self, environ, start_response):
         try:
             return self._handle_request(environ, start_response)
         except Exception:
             log.exception("Exception while handling request")
             appenlight.track_exception(environ)
             return HTTPInternalServerError()(environ, start_response)
         finally:
             meta.Session.remove()
 
     def _handle_request(self, environ, start_response):
 
         if not self._check_ssl(environ, start_response):
             reason = ('SSL required, while RhodeCode was unable '
                       'to detect this as SSL request')
             log.debug('User not allowed to proceed, %s', reason)
             return HTTPNotAcceptable(reason)(environ, start_response)
 
         ip_addr = get_ip_addr(environ)
         username = None
 
         # skip passing error to error controller
         environ['pylons.status_code_redirect'] = True
 
         # ======================================================================
         # EXTRACT REPOSITORY NAME FROM ENV
         # ======================================================================
         environ['PATH_INFO'] = self._get_by_id(environ['PATH_INFO'])
         repo_name = self._get_repository_name(environ)
         environ['REPO_NAME'] = repo_name
         log.debug('Extracted repo name is %s', repo_name)
 
         # check for type, presence in database and on filesystem
         if not self.is_valid_and_existing_repo(
                 repo_name, self.basepath, self.SCM):
             return HTTPNotFound()(environ, start_response)
 
         # ======================================================================
         # GET ACTION PULL or PUSH
         # ======================================================================
         action = self._get_action(environ)
 
         # ======================================================================
         # CHECK ANONYMOUS PERMISSION
         # ======================================================================
         if action in ['pull', 'push']:
             anonymous_user = User.get_default_user()
             username = anonymous_user.username
             if anonymous_user.active:
                 # ONLY check permissions if the user is activated
                 anonymous_perm = self._check_permission(
                     action, anonymous_user, repo_name, ip_addr)
             else:
                 anonymous_perm = False
 
             if not anonymous_user.active or not anonymous_perm:
                 if not anonymous_user.active:
                     log.debug('Anonymous access is disabled, running '
                               'authentication')
 
                 if not anonymous_perm:
                     log.debug('Not enough credentials to access this '
                               'repository as anonymous user')
 
                 username = None
                 # ==============================================================
                 # DEFAULT PERM FAILED OR ANONYMOUS ACCESS IS DISABLED SO WE
                 # NEED TO AUTHENTICATE AND ASK FOR AUTH USER PERMISSIONS
                 # ==============================================================
 
                 # try to auth based on environ, container auth methods
                 log.debug('Running PRE-AUTH for container based authentication')
                 pre_auth = authenticate(
                     '', '', environ, VCS_TYPE, registry=self.registry)
                 if pre_auth and pre_auth.get('username'):
                     username = pre_auth['username']
                 log.debug('PRE-AUTH got %s as username', username)
 
                 # If not authenticated by the container, running basic auth
                 if not username:
                     self.authenticate.realm = get_rhodecode_realm()
 
                     try:
                         result = self.authenticate(environ)
                     except (UserCreationError, NotAllowedToCreateUserError) as e:
                         log.error(e)
                         reason = safe_str(e)
                         return HTTPNotAcceptable(reason)(environ, start_response)
 
                     if isinstance(result, str):
                         AUTH_TYPE.update(environ, 'basic')
                         REMOTE_USER.update(environ, result)
                         username = result
                     else:
                         return result.wsgi_application(environ, start_response)
 
                 # ==============================================================
                 # CHECK PERMISSIONS FOR THIS REQUEST USING GIVEN USERNAME
                 # ==============================================================
                 user = User.get_by_username(username)
                 if not self.valid_and_active_user(user):
                     return HTTPForbidden()(environ, start_response)
                 username = user.username
                 user.update_lastactivity()
                 meta.Session().commit()
 
                 # check user attributes for password change flag
                 user_obj = user
                 if user_obj and user_obj.username != User.DEFAULT_USER and \
                         user_obj.user_data.get('force_password_change'):
                     reason = 'password change required'
                     log.debug('User not allowed to authenticate, %s', reason)
                     return HTTPNotAcceptable(reason)(environ, start_response)
 
                 # check permissions for this repository
                 perm = self._check_permission(action, user, repo_name, ip_addr)
                 if not perm:
                     return HTTPForbidden()(environ, start_response)
 
         # extras are injected into UI object and later available
         # in hooks executed by rhodecode
         check_locking = _should_check_locking(environ.get('QUERY_STRING'))
         extras = vcs_operation_context(
             environ, repo_name=repo_name, username=username,
             action=action, scm=self.SCM,
             check_locking=check_locking)
 
         # ======================================================================
         # REQUEST HANDLING
         # ======================================================================
         str_repo_name = safe_str(repo_name)
         repo_path = os.path.join(safe_str(self.basepath), str_repo_name)
         log.debug('Repository path is %s', repo_path)
 
         fix_PATH()
 
         log.info(
             '%s action on %s repo "%s" by "%s" from %s',
             action, self.SCM, str_repo_name, safe_str(username), ip_addr)
 
         return self._generate_vcs_response(
             environ, start_response, repo_path, repo_name, extras, action)
 
     @initialize_generator
     def _generate_vcs_response(
             self, environ, start_response, repo_path, repo_name, extras,
             action):
         """
         Returns a generator for the response content.
 
         This method is implemented as a generator, so that it can trigger
         the cache validation after all content sent back to the client. It
         also handles the locking exceptions which will be triggered when
         the first chunk is produced by the underlying WSGI application.
         """
         callback_daemon, extras = self._prepare_callback_daemon(extras)
         config = self._create_config(extras, repo_name)
         log.debug('HOOKS extras is %s', extras)
         app = self._create_wsgi_app(repo_path, repo_name, config)
 
         try:
             with callback_daemon:
                 try:
                     response = app(environ, start_response)
                 finally:
                     # This statement works together with the decorator
                     # "initialize_generator" above. The decorator ensures that
                     # we hit the first yield statement before the generator is
                     # returned back to the WSGI server. This is needed to
                     # ensure that the call to "app" above triggers the
                     # needed callback to "start_response" before the
                     # generator is actually used.
                     yield "__init__"
 
                 for chunk in response:
                     yield chunk
         except Exception as exc:
             # TODO: johbo: Improve "translating" back the exception.
             if getattr(exc, '_vcs_kind', None) == 'repo_locked':
                 exc = HTTPLockedRC(*exc.args)
                 _code = rhodecode.CONFIG.get('lock_ret_code')
                 log.debug('Repository LOCKED ret code %s!', (_code,))
             elif getattr(exc, '_vcs_kind', None) == 'requirement':
                 log.debug(
                     'Repository requires features unknown to this Mercurial')
                 exc = HTTPRequirementError(*exc.args)
             else:
                 raise
 
             for chunk in exc(environ, start_response):
                 yield chunk
         finally:
             # invalidate cache on push
-            if action == 'push':
-                self._invalidate_cache(repo_name)
+            try:
+                if action == 'push':
+                    self._invalidate_cache(repo_name)
+            finally:
+                meta.Session.remove()
 
     def _get_repository_name(self, environ):
         """Get repository name out of the environmnent
 
         :param environ: WSGI environment
         """
         raise NotImplementedError()
 
     def _get_action(self, environ):
         """Map request commands into a pull or push command.
 
         :param environ: WSGI environment
         """
         raise NotImplementedError()
 
     def _create_wsgi_app(self, repo_path, repo_name, config):
         """Return the WSGI app that will finally handle the request."""
         raise NotImplementedError()
 
     def _create_config(self, extras, repo_name):
         """Create a Pyro safe config representation."""
         raise NotImplementedError()
 
     def _prepare_callback_daemon(self, extras):
         return prepare_callback_daemon(
             extras, protocol=vcs_settings.HOOKS_PROTOCOL,
             use_direct_calls=vcs_settings.HOOKS_DIRECT_CALLS)
 
 
 def _should_check_locking(query_string):
     # this is kind of hacky, but due to how mercurial handles client-server
     # server see all operation on commit; bookmarks, phases and
     # obsolescence marker in different transaction, we don't want to check
     # locking on those
     return query_string not in ['cmd=listkeys']

rhodecode/tweens.py

@@ -1,86 +1,81 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2016  RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 
 import logging
 import pylons
 import rhodecode
 
 from pylons.i18n.translation import _get_translator
 from pylons.util import ContextObj
 from routes.util import URLGenerator
 
 from rhodecode.lib.base import attach_context_attributes, get_auth_user
 from rhodecode.model import meta
 
 log = logging.getLogger(__name__)
 
 
 def pylons_compatibility_tween_factory(handler, registry):
     def pylons_compatibility_tween(request):
         """
         While migrating from pylons to pyramid we need to call some pylons code
         from pyramid. For example while rendering an old template that uses the
         'c' or 'h' objects. This tween sets up the needed pylons globals.
         """
-        try:
-            config = rhodecode.CONFIG
-            environ = request.environ
-            session = request.session
-            session_key = (config['pylons.environ_config']
-                           .get('session', 'beaker.session'))
+        config = rhodecode.CONFIG
+        environ = request.environ
+        session = request.session
+        session_key = (config['pylons.environ_config']
+                       .get('session', 'beaker.session'))
 
-            # Setup pylons globals.
-            pylons.config._push_object(config)
-            pylons.request._push_object(request)
-            pylons.session._push_object(session)
-            environ[session_key] = session
-            pylons.url._push_object(URLGenerator(config['routes.map'],
-                                                 environ))
+        # Setup pylons globals.
+        pylons.config._push_object(config)
+        pylons.request._push_object(request)
+        pylons.session._push_object(session)
+        environ[session_key] = session
+        pylons.url._push_object(URLGenerator(config['routes.map'],
+                                             environ))
 
-            # TODO: Maybe we should use the language from pyramid.
-            translator = _get_translator(config.get('lang'))
-            pylons.translator._push_object(translator)
-
-            # Get the rhodecode auth user object and make it available.
-            auth_user = get_auth_user(environ)
-            request.user = auth_user
-            environ['rc_auth_user'] = auth_user
+        # TODO: Maybe we should use the language from pyramid.
+        translator = _get_translator(config.get('lang'))
+        pylons.translator._push_object(translator)
 
-            # Setup the pylons context object ('c')
-            context = ContextObj()
-            context.rhodecode_user = auth_user
-            attach_context_attributes(context, request)
-            pylons.tmpl_context._push_object(context)
-            return handler(request)
-        finally:
-            # Dispose current database session and rollback uncommitted
-            # transactions.
-            meta.Session.remove()
+        # Get the rhodecode auth user object and make it available.
+        auth_user = get_auth_user(environ)
+        request.user = auth_user
+        environ['rc_auth_user'] = auth_user
+
+        # Setup the pylons context object ('c')
+        context = ContextObj()
+        context.rhodecode_user = auth_user
+        attach_context_attributes(context, request)
+        pylons.tmpl_context._push_object(context)
+        return handler(request)
 
     return pylons_compatibility_tween
 
 
 def includeme(config):
     config.add_subscriber('rhodecode.subscribers.add_renderer_globals',
                           'pyramid.events.BeforeRender')
     config.add_subscriber('rhodecode.subscribers.add_localizer',
                           'pyramid.events.NewRequest')
     config.add_tween('rhodecode.tweens.pylons_compatibility_tween_factory')