##// END OF EJS Templates
http-proto: in case incoming requests come in as chunked stream the data to VCSServer....
marcink -
r1434:59cf3775 stable
parent child Browse files
Show More
@@ -1,137 +1,143 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2
2
3 # Copyright (C) 2014-2017 RhodeCode GmbH
3 # Copyright (C) 2014-2017 RhodeCode GmbH
4 #
4 #
5 # This program is free software: you can redistribute it and/or modify
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU Affero General Public License, version 3
6 # it under the terms of the GNU Affero General Public License, version 3
7 # (only), as published by the Free Software Foundation.
7 # (only), as published by the Free Software Foundation.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU Affero General Public License
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 #
16 #
17 # This program is dual-licensed. If you wish to learn more about the
17 # This program is dual-licensed. If you wish to learn more about the
18 # RhodeCode Enterprise Edition, including its added features, Support services,
18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20
20
21 """
21 """
22 Implementation of the scm_app interface using raw HTTP communication.
22 Implementation of the scm_app interface using raw HTTP communication.
23 """
23 """
24
24
25 import base64
25 import base64
26 import logging
26 import logging
27 import urlparse
27 import urlparse
28 import wsgiref.util
28 import wsgiref.util
29
29
30 import msgpack
30 import msgpack
31 import requests
31 import requests
32 import webob.request
32 import webob.request
33
33
34 import rhodecode
34 import rhodecode
35
35
36
36
37 log = logging.getLogger(__name__)
37 log = logging.getLogger(__name__)
38
38
39
39
40 def create_git_wsgi_app(repo_path, repo_name, config):
40 def create_git_wsgi_app(repo_path, repo_name, config):
41 url = _vcs_streaming_url() + 'git/'
41 url = _vcs_streaming_url() + 'git/'
42 return VcsHttpProxy(url, repo_path, repo_name, config)
42 return VcsHttpProxy(url, repo_path, repo_name, config)
43
43
44
44
45 def create_hg_wsgi_app(repo_path, repo_name, config):
45 def create_hg_wsgi_app(repo_path, repo_name, config):
46 url = _vcs_streaming_url() + 'hg/'
46 url = _vcs_streaming_url() + 'hg/'
47 return VcsHttpProxy(url, repo_path, repo_name, config)
47 return VcsHttpProxy(url, repo_path, repo_name, config)
48
48
49
49
50 def _vcs_streaming_url():
50 def _vcs_streaming_url():
51 template = 'http://{}/stream/'
51 template = 'http://{}/stream/'
52 return template.format(rhodecode.CONFIG['vcs.server'])
52 return template.format(rhodecode.CONFIG['vcs.server'])
53
53
54
54
55 # TODO: johbo: Avoid the global.
55 # TODO: johbo: Avoid the global.
56 session = requests.Session()
56 session = requests.Session()
57 # Requests speedup, avoid reading .netrc and similar
57 # Requests speedup, avoid reading .netrc and similar
58 session.trust_env = False
58 session.trust_env = False
59
59
60
60
61 class VcsHttpProxy(object):
61 class VcsHttpProxy(object):
62 """
62 """
63 A WSGI application which proxies vcs requests.
63 A WSGI application which proxies vcs requests.
64
64
65 The goal is to shuffle the data around without touching it. The only
65 The goal is to shuffle the data around without touching it. The only
66 exception is the extra data from the config object which we send to the
66 exception is the extra data from the config object which we send to the
67 server as well.
67 server as well.
68 """
68 """
69
69
70 def __init__(self, url, repo_path, repo_name, config):
70 def __init__(self, url, repo_path, repo_name, config):
71 """
71 """
72 :param str url: The URL of the VCSServer to call.
72 :param str url: The URL of the VCSServer to call.
73 """
73 """
74 self._url = url
74 self._url = url
75 self._repo_name = repo_name
75 self._repo_name = repo_name
76 self._repo_path = repo_path
76 self._repo_path = repo_path
77 self._config = config
77 self._config = config
78 log.debug(
78 log.debug(
79 "Creating VcsHttpProxy for repo %s, url %s",
79 "Creating VcsHttpProxy for repo %s, url %s",
80 repo_name, url)
80 repo_name, url)
81
81
82 def __call__(self, environ, start_response):
82 def __call__(self, environ, start_response):
83 config = msgpack.packb(self._config)
83 config = msgpack.packb(self._config)
84 request = webob.request.Request(environ)
84 request = webob.request.Request(environ)
85 request_headers = request.headers
85 request_headers = request.headers
86 request_headers.update({
86 request_headers.update({
87 # TODO: johbo: Remove this, rely on URL path only
87 # TODO: johbo: Remove this, rely on URL path only
88 'X-RC-Repo-Name': self._repo_name,
88 'X-RC-Repo-Name': self._repo_name,
89 'X-RC-Repo-Path': self._repo_path,
89 'X-RC-Repo-Path': self._repo_path,
90 'X-RC-Path-Info': environ['PATH_INFO'],
90 'X-RC-Path-Info': environ['PATH_INFO'],
91 # TODO: johbo: Avoid encoding and put this into payload?
91 # TODO: johbo: Avoid encoding and put this into payload?
92 'X-RC-Repo-Config': base64.b64encode(config),
92 'X-RC-Repo-Config': base64.b64encode(config),
93 'X-RC-Locked-Status-Code': rhodecode.CONFIG.get('lock_ret_code')
93 'X-RC-Locked-Status-Code': rhodecode.CONFIG.get('lock_ret_code')
94 })
94 })
95
95
96 data = environ['wsgi.input'].read()
97 method = environ['REQUEST_METHOD']
96 method = environ['REQUEST_METHOD']
98
97
99 # Preserve the query string
98 # Preserve the query string
100 url = self._url
99 url = self._url
101 url = urlparse.urljoin(url, self._repo_name)
100 url = urlparse.urljoin(url, self._repo_name)
102 if environ.get('QUERY_STRING'):
101 if environ.get('QUERY_STRING'):
103 url += '?' + environ['QUERY_STRING']
102 url += '?' + environ['QUERY_STRING']
104
103
105 response = session.request(
104 response = session.request(
106 method, url,
105 method, url,
107 data=data,
106 data=_maybe_stream_request(environ),
108 headers=request_headers,
107 headers=request_headers,
109 stream=True)
108 stream=True)
110
109
111 # Preserve the headers of the response, except hop_by_hop ones
110 # Preserve the headers of the response, except hop_by_hop ones
112 response_headers = [
111 response_headers = [
113 (h, v) for h, v in response.headers.items()
112 (h, v) for h, v in response.headers.items()
114 if not wsgiref.util.is_hop_by_hop(h)
113 if not wsgiref.util.is_hop_by_hop(h)
115 ]
114 ]
116
115
117 # Build status argument for start_reponse callable.
116 # Build status argument for start_reponse callable.
118 status = '{status_code} {reason_phrase}'.format(
117 status = '{status_code} {reason_phrase}'.format(
119 status_code=response.status_code,
118 status_code=response.status_code,
120 reason_phrase=response.reason)
119 reason_phrase=response.reason)
121
120
122 start_response(status, response_headers)
121 start_response(status, response_headers)
123 return _maybe_stream(response)
122 return _maybe_stream_response(response)
124
123
125
124
126 def _maybe_stream(response):
125 def _maybe_stream_request(environ):
126 if environ.get('HTTP_TRANSFER_ENCODING', '') == 'chunked':
127 return environ['wsgi.input']
128 else:
129 return environ['wsgi.input'].read()
130
131
132 def _maybe_stream_response(response):
127 """
133 """
128 Try to generate chunks from the response if it is chunked.
134 Try to generate chunks from the response if it is chunked.
129 """
135 """
130 if _is_chunked(response):
136 if _is_chunked(response):
131 return response.raw.read_chunked()
137 return response.raw.read_chunked()
132 else:
138 else:
133 return [response.content]
139 return [response.content]
134
140
135
141
136 def _is_chunked(response):
142 def _is_chunked(response):
137 return response.headers.get('Transfer-Encoding', '') == 'chunked'
143 return response.headers.get('Transfer-Encoding', '') == 'chunked'
General Comments 0
You need to be logged in to leave comments. Login now