hg: Redirect Mercurial stdout/stderr to logging when running as WSGI

Any "console" output from Mercurial when Kallithea is running from WSGI
should end up in Kallithea's logs. That seems like a nice general feature.

This will however also solve another rare but more critical problem:

Mercurial is writing to sys.stdout / sys.stderr, using several layers of
wrapping. Since Mercurial 5.5 (with
https://repo.mercurial-scm.org/hg/rev/8e04607023e5 ), all writes are given a
memoryview.

Apache httpd mod_wsgi is invoking the WSGI with a custom mod_wsgi.Log injected
in sys.stdout / sys.stderr . This logger can however not handle memoryview -
https://github.com/GrahamDumpleton/mod_wsgi/issues/863 .

kallithea/config/application.py

@@ -1,74 +1,76
 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """WSGI middleware initialization for the Kallithea application."""
 
 from kallithea.config.app_cfg import base_config
 from kallithea.config.middleware.https_fixup import HttpsFixup
 from kallithea.config.middleware.permanent_repo_url import PermanentRepoUrl
 from kallithea.config.middleware.simplegit import SimpleGit
 from kallithea.config.middleware.simplehg import SimpleHg
 from kallithea.config.middleware.wrapper import RequestWrapper
 from kallithea.lib.utils2 import asbool
+from kallithea.lib.vcs.utils import hgcompat
 
 
 __all__ = ['make_app']
 
 
 def wrap_app(app):
     """Wrap the TG WSGI application in Kallithea middleware"""
     config = app.config
 
     # we want our low level middleware to get to the request ASAP. We don't
     # need any stack middleware in them - especially no StatusCodeRedirect buffering
     app = SimpleHg(app, config)
     app = SimpleGit(app, config)
 
     # Enable https redirects based on HTTP_X_URL_SCHEME set by proxy
     if config.get('url_scheme_variable') or asbool(config.get('force_https')) or asbool(config.get('use_htsts')):
         app = HttpsFixup(app, config)
 
     app = PermanentRepoUrl(app, config)
 
     # Optional and undocumented wrapper - gives more verbose request/response logging, but has a slight overhead
     if asbool(config.get('use_wsgi_wrapper')):
         app = RequestWrapper(app, config)
 
     return app
 
 
 def make_app(global_conf, **app_conf):
     """Return WSGI app with logging Mercurial stdout/stderr - to be used as
     Paste or mod_wsgi entry point"""
+    hgcompat.redirect_stdio_to_logging()
     return make_app_raw(global_conf, **app_conf)
 
 
 def make_app_raw(global_conf, **app_conf):
     """
     Set up Kallithea with the settings found in the PasteDeploy configuration
     file used.
 
     :param global_conf: The global settings for Kallithea (those
         defined under the ``[DEFAULT]`` section).
     :return: The Kallithea application with all the relevant middleware
         loaded.
 
     This is the PasteDeploy factory for the Kallithea application.
 
     ``app_conf`` contains all the application-specific settings (those defined
     under ``[app:main]``.
     """
     assert app_conf.get('sqlalchemy.url')  # must be called with a Kallithea .ini file, which for example must have this config option
     assert global_conf.get('here') and global_conf.get('__file__')  # app config should be initialized the paste way ...
 
     return base_config.make_wsgi_app(global_conf, app_conf, wrap_app=wrap_app)

kallithea/lib/vcs/utils/hgcompat.py

@@ -1,17 +1,47
 """
 Mercurial libs compatibility
 """
 
+import logging
+
 import mercurial.encoding
 import mercurial.localrepo
 
 
+class MercurialStdLogger:
+    def __init__(self, logger):
+        self.logger = logger
+
+    def write(self, message):
+        try:
+            self.logger(message.decode().rstrip())
+        except:
+            self.logger(message)
+
+    def flush(self):
+        pass
+
 def monkey_do():
     """Apply some Mercurial monkey patching"""
     # workaround for 3.3 94ac64bcf6fe and not calling largefiles reposetup correctly, and test_archival failing
     mercurial.localrepo.localrepository._lfstatuswriters = [lambda *msg, **opts: None]
     # 3.5 7699d3212994 added the invariant that repo.lfstatus must exist before hitting overridearchive
     mercurial.localrepo.localrepository.lfstatus = False
 
     # Minimize potential impact from custom configuration
     mercurial.encoding.environ[b'HGPLAIN'] = b'1'
+
+
+hglog = logging.getLogger("mercurial")
+
+
+def redirect_stdio_to_logging():
+    # Capture Mercurial stdout/stderr and send to a 'mercurial' logger
+    try:
+        import mercurial.utils.procutil as procutil
+        if not isinstance(procutil.stdout, MercurialStdLogger):
+            procutil.stdout = MercurialStdLogger(hglog.info)
+        if not isinstance(procutil.stderr, MercurialStdLogger):
+            procutil.stderr = MercurialStdLogger(hglog.warning)
+    except Exception as e:
+        hglog.error("Exception installing procutil stdout/stderr: %s", e)