pyro4: Add the custom header `X-RhodeCode-Backend` to the pyro4 backend responses.

This custom header sets the SCM backend which is in use. It is used to identify
VCS responses in the error handler. VCS responses are skipped during error handling.

rhodecode/lib/middleware/simplegit.py

@@ -1,84 +1,85 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2016  RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 SimpleGit middleware for handling git protocol request (push/clone etc.)
 It's implemented with basic auth function
 """
 import re
 import logging
 import urlparse
 
 import rhodecode
 from rhodecode.lib import utils2
 from rhodecode.lib.middleware import simplevcs
 
 log = logging.getLogger(__name__)
 
 
 GIT_PROTO_PAT = re.compile(
     r'^/(.+)/(info/refs|git-upload-pack|git-receive-pack)')
 
 
 class SimpleGit(simplevcs.SimpleVCS):
 
     SCM = 'git'
 
     def _get_repository_name(self, environ):
         """
         Gets repository name out of PATH_INFO header
 
         :param environ: environ where PATH_INFO is stored
         """
         repo_name = GIT_PROTO_PAT.match(environ['PATH_INFO']).group(1)
         return repo_name
 
     _ACTION_MAPPING = {
         'git-receive-pack': 'push',
         'git-upload-pack': 'pull',
     }
 
     def _get_action(self, environ):
         """
         Maps git request commands into a pull or push command.
         In case of unknown/unexpected data, it returns 'pull' to be safe.
 
         :param environ:
         """
         path = environ['PATH_INFO']
 
         if path.endswith('/info/refs'):
             query = urlparse.parse_qs(environ['QUERY_STRING'])
             service_cmd = query.get('service', [''])[0]
             return self._ACTION_MAPPING.get(service_cmd, 'pull')
         elif path.endswith('/git-receive-pack'):
             return 'push'
         elif path.endswith('/git-upload-pack'):
             return 'pull'
 
         return 'pull'
 
     def _create_wsgi_app(self, repo_path, repo_name, config):
-        return self.scm_app.create_git_wsgi_app(repo_path, repo_name, config)
+        return self.scm_app.create_git_wsgi_app(
+            repo_path, repo_name, config, self.SCM)
 
     def _create_config(self, extras, repo_name):
         extras['git_update_server_info'] = utils2.str2bool(
             rhodecode.CONFIG.get('git_update_server_info'))
         return extras

rhodecode/lib/middleware/simplehg.py

@@ -1,80 +1,81 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2016  RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 SimpleHG middleware for handling mercurial protocol request
 (push/clone etc.). It's implemented with basic auth function
 """
 
 import logging
 import urlparse
 
 from rhodecode.lib import utils
 from rhodecode.lib.ext_json import json
 from rhodecode.lib.middleware import simplevcs
 
 log = logging.getLogger(__name__)
 
 
 class SimpleHg(simplevcs.SimpleVCS):
 
     SCM = 'hg'
 
     def _get_repository_name(self, environ):
         """
         Gets repository name out of PATH_INFO header
 
         :param environ: environ where PATH_INFO is stored
         """
         return environ['PATH_INFO'].strip('/')
 
     _ACTION_MAPPING = {
         'changegroup': 'pull',
         'changegroupsubset': 'pull',
         'getbundle': 'pull',
         'stream_out': 'pull',
         'listkeys': 'pull',
         'unbundle': 'push',
         'pushkey': 'push',
     }
 
     def _get_action(self, environ):
         """
         Maps mercurial request commands into a pull or push command.
         In case of unknown/unexpected data, it returns 'pull' to be safe.
 
         :param environ:
         """
         query = urlparse.parse_qs(environ['QUERY_STRING'],
                                   keep_blank_values=True)
         if 'cmd' in query:
             cmd = query['cmd'][0]
             return self._ACTION_MAPPING.get(cmd, 'pull')
 
         return 'pull'
 
     def _create_wsgi_app(self, repo_path, repo_name, config):
-        return self.scm_app.create_hg_wsgi_app(repo_path, repo_name, config)
+        return self.scm_app.create_hg_wsgi_app(
+            repo_path, repo_name, config, self.SCM)
 
     def _create_config(self, extras, repo_name):
         config = utils.make_db_config(repo=repo_name)
         config.set('rhodecode', 'RC_SCM_DATA', json.dumps(extras))
 
         return config.serialize()

rhodecode/lib/middleware/utils/scm_app.py

@@ -1,63 +1,63 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2012-2016  RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Equivalent of rhodecode.lib.middleware.scm_app but using remote apps.
 """
 
 import logging
 
 from rhodecode.lib.middleware.utils import wsgi_app_caller_client
 
 
 log = logging.getLogger(__name__)
 
 
 HG_REMOTE_WSGI = None
 GIT_REMOTE_WSGI = None
 
 
-def create_git_wsgi_app(repo_path, repo_name, config):
+def create_git_wsgi_app(repo_path, repo_name, config, backend):
     """
     Return a WSGI app backed by a remote app to handle Git.
 
     config is a dictionary holding the extras.
     """
     factory = GIT_REMOTE_WSGI
     if not factory:
         log.error('Pyro server has not been initialized yet')
 
     return wsgi_app_caller_client.RemoteAppCaller(
-        factory, repo_path, repo_name, config)
+        factory, backend, repo_path, repo_name, config)
 
 
-def create_hg_wsgi_app(repo_path, repo_name, config):
+def create_hg_wsgi_app(repo_path, repo_name, config, backend):
     """
     Return a WSGI app backed by a remote app to handle Mercurial.
 
     config is a list of 3-item tuples representing a ConfigObject (it is the
     serialized version of the config object).
     """
     factory = HG_REMOTE_WSGI
     if not factory:
         log.error('Pyro server has not been initialized yet')
 
     return wsgi_app_caller_client.RemoteAppCaller(
-        factory, repo_path, repo_name, config)
+        factory, backend, repo_path, repo_name, config)

rhodecode/lib/middleware/utils/scm_app_http.py

@@ -1,140 +1,140 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2014-2016  RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Implementation of the scm_app interface using raw HTTP communication.
 """
 
 import base64
 import logging
 import urlparse
 import wsgiref.util
 
 import msgpack
 import requests
 import webob.request
 
 import rhodecode
 
 
 log = logging.getLogger(__name__)
 
 
-def create_git_wsgi_app(repo_path, repo_name, config):
+def create_git_wsgi_app(repo_path, repo_name, config, backend):
     url = _vcs_streaming_url() + 'git/'
-    return VcsHttpProxy(url, repo_path, repo_name, config, 'git')
+    return VcsHttpProxy(url, repo_path, repo_name, config, backend)
 
 
-def create_hg_wsgi_app(repo_path, repo_name, config):
+def create_hg_wsgi_app(repo_path, repo_name, config, backend):
     url = _vcs_streaming_url() + 'hg/'
-    return VcsHttpProxy(url, repo_path, repo_name, config, 'hg')
+    return VcsHttpProxy(url, repo_path, repo_name, config, backend)
 
 
 def _vcs_streaming_url():
     template = 'http://{}/stream/'
     return template.format(rhodecode.CONFIG['vcs.server'])
 
 
 # TODO: johbo: Avoid the global.
 session = requests.Session()
 # Requests speedup, avoid reading .netrc and similar
 session.trust_env = False
 
 
 class VcsHttpProxy(object):
     """
     A WSGI application which proxies vcs requests.
 
     The goal is to shuffle the data around without touching it. The only
     exception is the extra data from the config object which we send to the
     server as well.
     """
 
     def __init__(self, url, repo_path, repo_name, config, backend):
         """
         :param str url: The URL of the VCSServer to call.
         """
         self._url = url
         self._repo_name = repo_name
         self._repo_path = repo_path
         self._config = config
         self._backend = backend
         log.debug(
             "Creating VcsHttpProxy for repo %s, url %s",
             repo_name, url)
 
     def __call__(self, environ, start_response):
         status = '200 OK'
 
         config = msgpack.packb(self._config)
         request = webob.request.Request(environ)
         request_headers = request.headers
         request_headers.update({
             # TODO: johbo: Remove this, rely on URL path only
             'X-RC-Repo-Name': self._repo_name,
             'X-RC-Repo-Path': self._repo_path,
             'X-RC-Path-Info': environ['PATH_INFO'],
             # TODO: johbo: Avoid encoding and put this into payload?
             'X-RC-Repo-Config': base64.b64encode(config),
         })
 
         data = environ['wsgi.input'].read()
         method = environ['REQUEST_METHOD']
 
         # Preserve the query string
         url = self._url
         url = urlparse.urljoin(url, self._repo_name)
         if environ.get('QUERY_STRING'):
             url += '?' + environ['QUERY_STRING']
 
         response = session.request(
             method, url,
             data=data,
             headers=request_headers,
             stream=True)
 
         # Preserve the headers of the response, except hop_by_hop ones
         response_headers = [
             (h, v) for h, v in response.headers.items()
             if not wsgiref.util.is_hop_by_hop(h)
         ]
 
         # Add custom response header to indicate that this is a VCS response
         # and which backend is used.
         response_headers.append(('X-RhodeCode-Backend', self._backend))
 
         # TODO: johbo: Better way to get the status including text?
         status = str(response.status_code)
         start_response(status, response_headers)
         return _maybe_stream(response)
 
 
 def _maybe_stream(response):
     """
     Try to generate chunks from the response if it is chunked.
     """
     if _is_chunked(response):
         return response.raw.read_chunked()
     else:
         return [response.content]
 
 
 def _is_chunked(response):
     return response.headers.get('Transfer-Encoding', '') == 'chunked'

rhodecode/lib/middleware/utils/wsgi_app_caller_client.py

@@ -1,98 +1,104 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2012-2016  RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Utility to call a WSGI app wrapped in a WSGIAppCaller object.
 """
 
 import logging
 
 from Pyro4.errors import ConnectionClosedError
 
 
 log = logging.getLogger(__name__)
 
 
 def _get_clean_environ(environ):
     """Return a copy of the WSGI environment without wsgi.* keys.
 
     It also omits any non-string values.
 
     :param environ: WSGI environment to clean
     :type environ: dict
 
     :returns: WSGI environment to pass to WSGIAppCaller.handle.
     :rtype: dict
     """
     clean_environ = dict(
         (k, v) for k, v in environ.iteritems()
         if type(v) == str and type(k) == str and not k.startswith('wsgi.')
     )
 
     return clean_environ
 
 
 # pylint: disable=too-few-public-methods
 class RemoteAppCaller(object):
     """Create and calls a remote WSGI app using the given factory.
 
     It first cleans the environment, so as to reduce the data transferred.
     """
 
-    def __init__(self, remote_wsgi, *args, **kwargs):
+    def __init__(self, remote_wsgi, backend, *args, **kwargs):
         """
         :param remote_wsgi: The remote wsgi object that creates a
           WSGIAppCaller. This object
           has to have a handle method, with the signature:
           handle(environ, start_response, *args, **kwargs)
+        :param backend: Key (str) of the SCM backend that is in use.
         :param args: args to be passed to the app creation
         :param kwargs: kwargs to be passed to the app creation
         """
         self._remote_wsgi = remote_wsgi
+        self._backend = backend
         self._args = args
         self._kwargs = kwargs
 
     def __call__(self, environ, start_response):
         """
         :param environ: WSGI environment with which the app will be run
         :type environ: dict
         :param start_response: callable of WSGI protocol
         :type start_response: callable
 
         :returns: an iterable with the data returned by the app
         :rtype: iterable<str>
         """
         log.debug("Forwarding WSGI request via proxy %s", self._remote_wsgi)
         input_data = environ['wsgi.input'].read()
         clean_environ = _get_clean_environ(environ)
 
         try:
             data, status, headers = self._remote_wsgi.handle(
                 clean_environ, input_data, *self._args, **self._kwargs)
         except ConnectionClosedError:
             log.debug('Remote Pyro Server ConnectionClosedError')
             self._remote_wsgi._pyroReconnect(tries=15)
             data, status, headers = self._remote_wsgi.handle(
                 clean_environ, input_data, *self._args, **self._kwargs)
 
+        # Add custom response header to indicate that this is a VCS response
+        # and which backend is used.
+        headers.append(('X-RhodeCode-Backend', self._backend))
+
         log.debug("Got result from proxy, returning to WSGI container")
         start_response(status, headers)
 
         return data

rhodecode/tests/lib/middleware/mock_scm_app.py

@@ -1,41 +1,41 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2016  RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Provides a mock implementation of the scm_app module.
 
 It resembles the same API as :mod:`rhodecode.lib.middleware.utils.scm_app`
 for testing purposes.
 """
 import mock
 
 
-def create_git_wsgi_app(repo_path, repo_name, config):
+def create_git_wsgi_app(repo_path, repo_name, config, backend):
     return mock_git_wsgi
 
 
-def create_hg_wsgi_app(repo_path, repo_name, config):
+def create_hg_wsgi_app(repo_path, repo_name, config, backend):
     return mock_hg_wsgi
 
 
 # Utilities to ease testing
 
 mock_hg_wsgi = mock.Mock()
 mock_git_wsgi = mock.Mock()