db: move Session.remove to outer wsgi layer and also add it

to hooks daemon to avoid leaving connections open in the db pool
fixes  #4173, refs  #4166

rhodecode/config/middleware.py

@@ -39,6 +39,7 b' from routes.middleware import RoutesMidd'
 import routes.util
 
 import rhodecode
+from rhodecode.model import meta
 from rhodecode.config import patches
 from rhodecode.config.routing import STATIC_FILE_PREFIX
 from rhodecode.config.environment import (
@@ -158,6 +159,10 b' def make_pyramid_app(global_config, **se'
     pyramid_app = config.make_wsgi_app()
     pyramid_app = wrap_app_in_wsgi_middlewares(pyramid_app, config)
     pyramid_app.config = config
+
+    # creating the app uses a connection - return it after we are done
+    meta.Session.remove()
+
     return pyramid_app
 
 
@@ -374,7 +379,25 b' def wrap_app_in_wsgi_middlewares(pyramid'
         pyramid_app = make_gzip_middleware(
             pyramid_app, settings, compress_level=1)
 
-    return pyramid_app
+
+    # this should be the outer most middleware in the wsgi stack since
+    # middleware like Routes make database calls
+    def pyramid_app_with_cleanup(environ, start_response):
+        try:
+            return pyramid_app(environ, start_response)
+        finally:
+            # Dispose current database session and rollback uncommitted
+            # transactions.
+            meta.Session.remove()
+
+            # In a single threaded mode server, on non sqlite db we should have
+            # '0 Current Checked out connections' at the end of a request,
+            # if not, then something, somewhere is leaving a connection open
+            pool = meta.Base.metadata.bind.engine.pool
+            log.debug('sa pool status: %s', pool.status())
+
+
+    return pyramid_app_with_cleanup
 
 
 def sanitize_settings_and_apply_defaults(settings):

rhodecode/lib/hooks_daemon.py

@@ -30,6 +30,7 b' import Pyro4'
 import pylons
 import rhodecode
 
+from rhodecode.model import meta
 from rhodecode.lib import hooks_base
 from rhodecode.lib.utils2 import (
     AttributeDict, safe_str, get_routes_generator_for_server_url)
@@ -64,7 +65,10 b' class HooksHttpHandler(BaseHTTPRequestHa'
 
     def _call_hook(self, method, extras):
         hooks = Hooks()
-        result = getattr(hooks, method)(extras)
+        try:
+            result = getattr(hooks, method)(extras)
+        finally:
+            meta.Session.remove()
         return result
 
     def log_message(self, format, *args):

rhodecode/lib/middleware/simplevcs.py

@@ -406,8 +406,11 b' class SimpleVCS(object):'
                 yield chunk
         finally:
             # invalidate cache on push
-            if action == 'push':
-                self._invalidate_cache(repo_name)
+            try:
+                if action == 'push':
+                    self._invalidate_cache(repo_name)
+            finally:
+                meta.Session.remove()
 
     def _get_repository_name(self, environ):
         """Get repository name out of the environmnent

rhodecode/tweens.py

@@ -40,40 +40,35 b' def pylons_compatibility_tween_factory(h'
         from pyramid. For example while rendering an old template that uses the
         'c' or 'h' objects. This tween sets up the needed pylons globals.
         """
-        try:
-            config = rhodecode.CONFIG
-            environ = request.environ
-            session = request.session
-            session_key = (config['pylons.environ_config']
-                           .get('session', 'beaker.session'))
+        config = rhodecode.CONFIG
+        environ = request.environ
+        session = request.session
+        session_key = (config['pylons.environ_config']
+                       .get('session', 'beaker.session'))
 
-            # Setup pylons globals.
-            pylons.config._push_object(config)
-            pylons.request._push_object(request)
-            pylons.session._push_object(session)
-            environ[session_key] = session
-            pylons.url._push_object(URLGenerator(config['routes.map'],
-                                                 environ))
+        # Setup pylons globals.
+        pylons.config._push_object(config)
+        pylons.request._push_object(request)
+        pylons.session._push_object(session)
+        environ[session_key] = session
+        pylons.url._push_object(URLGenerator(config['routes.map'],
+                                             environ))
 
-            # TODO: Maybe we should use the language from pyramid.
-            translator = _get_translator(config.get('lang'))
-            pylons.translator._push_object(translator)
-
-            # Get the rhodecode auth user object and make it available.
-            auth_user = get_auth_user(environ)
-            request.user = auth_user
-            environ['rc_auth_user'] = auth_user
+        # TODO: Maybe we should use the language from pyramid.
+        translator = _get_translator(config.get('lang'))
+        pylons.translator._push_object(translator)
 
-            # Setup the pylons context object ('c')
-            context = ContextObj()
-            context.rhodecode_user = auth_user
-            attach_context_attributes(context, request)
-            pylons.tmpl_context._push_object(context)
-            return handler(request)
-        finally:
-            # Dispose current database session and rollback uncommitted
-            # transactions.
-            meta.Session.remove()
+        # Get the rhodecode auth user object and make it available.
+        auth_user = get_auth_user(environ)
+        request.user = auth_user
+        environ['rc_auth_user'] = auth_user
+
+        # Setup the pylons context object ('c')
+        context = ContextObj()
+        context.rhodecode_user = auth_user
+        attach_context_attributes(context, request)
+        pylons.tmpl_context._push_object(context)
+        return handler(request)
 
     return pylons_compatibility_tween