db: move Session.remove to outer wsgi layer and also add it

to hooks daemon to avoid leaving connections open in the db pool
fixes  #4173, refs  #4166

rhodecode/config/middleware.py

@@ -39,6 +39,7 b' from routes.middleware import RoutesMidd'
 import routes.util
 
 import rhodecode
+from rhodecode.model import meta
 from rhodecode.config import patches
 from rhodecode.config.routing import STATIC_FILE_PREFIX
 from rhodecode.config.environment import (
@@ -158,6 +159,10 b' def make_pyramid_app(global_config, **se'
     pyramid_app = config.make_wsgi_app()
     pyramid_app = wrap_app_in_wsgi_middlewares(pyramid_app, config)
     pyramid_app.config = config
+
+    # creating the app uses a connection - return it after we are done
+    meta.Session.remove()
+
     return pyramid_app
 
 
@@ -374,7 +379,25 b' def wrap_app_in_wsgi_middlewares(pyramid'
         pyramid_app = make_gzip_middleware(
             pyramid_app, settings, compress_level=1)
 
-    return pyramid_app
+
+    # this should be the outer most middleware in the wsgi stack since
+    # middleware like Routes make database calls
+    def pyramid_app_with_cleanup(environ, start_response):
+        try:
+            return pyramid_app(environ, start_response)
+        finally:
+            # Dispose current database session and rollback uncommitted
+            # transactions.
+            meta.Session.remove()
+
+            # In a single threaded mode server, on non sqlite db we should have
+            # '0 Current Checked out connections' at the end of a request,
+            # if not, then something, somewhere is leaving a connection open
+            pool = meta.Base.metadata.bind.engine.pool
+            log.debug('sa pool status: %s', pool.status())
+
+
+    return pyramid_app_with_cleanup
 
 
 def sanitize_settings_and_apply_defaults(settings):

rhodecode/lib/hooks_daemon.py

@@ -30,6 +30,7 b' import Pyro4'
 import pylons
 import rhodecode
 
+from rhodecode.model import meta
 from rhodecode.lib import hooks_base
 from rhodecode.lib.utils2 import (
     AttributeDict, safe_str, get_routes_generator_for_server_url)
@@ -64,7 +65,10 b' class HooksHttpHandler(BaseHTTPRequestHa'
 
     def _call_hook(self, method, extras):
         hooks = Hooks()
+        try:
         result = getattr(hooks, method)(extras)
+        finally:
+            meta.Session.remove()
         return result
 
     def log_message(self, format, *args):

rhodecode/lib/middleware/simplevcs.py

@@ -406,8 +406,11 b' class SimpleVCS(object):'
                 yield chunk
         finally:
             # invalidate cache on push
+            try:
             if action == 'push':
                 self._invalidate_cache(repo_name)
+            finally:
+                meta.Session.remove()
 
     def _get_repository_name(self, environ):
         """Get repository name out of the environmnent

rhodecode/tweens.py

@@ -40,7 +40,6 b' def pylons_compatibility_tween_factory(h'
         from pyramid. For example while rendering an old template that uses the
         'c' or 'h' objects. This tween sets up the needed pylons globals.
         """
-        try:
             config = rhodecode.CONFIG
             environ = request.environ
             session = request.session
@@ -70,10 +69,6 b' def pylons_compatibility_tween_factory(h'
             attach_context_attributes(context, request)
             pylons.tmpl_context._push_object(context)
             return handler(request)
-        finally:
-            # Dispose current database session and rollback uncommitted
-            # transactions.
-            meta.Session.remove()
 
     return pylons_compatibility_tween