##// END OF EJS Templates
http-app: simplify detection of chunked encoding....
marcink -
r1643:ae28d33d default
parent child Browse files
Show More
@@ -1,162 +1,158 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2
2
3 # Copyright (C) 2014-2017 RhodeCode GmbH
3 # Copyright (C) 2014-2017 RhodeCode GmbH
4 #
4 #
5 # This program is free software: you can redistribute it and/or modify
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU Affero General Public License, version 3
6 # it under the terms of the GNU Affero General Public License, version 3
7 # (only), as published by the Free Software Foundation.
7 # (only), as published by the Free Software Foundation.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU Affero General Public License
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 #
16 #
17 # This program is dual-licensed. If you wish to learn more about the
17 # This program is dual-licensed. If you wish to learn more about the
18 # RhodeCode Enterprise Edition, including its added features, Support services,
18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20
20
21 """
21 """
22 Implementation of the scm_app interface using raw HTTP communication.
22 Implementation of the scm_app interface using raw HTTP communication.
23 """
23 """
24
24
25 import base64
25 import base64
26 import logging
26 import logging
27 import urlparse
27 import urlparse
28 import wsgiref.util
28 import wsgiref.util
29
29
30 import msgpack
30 import msgpack
31 import requests
31 import requests
32 import webob.request
32 import webob.request
33
33
34 import rhodecode
34 import rhodecode
35
35
36
36
37 log = logging.getLogger(__name__)
37 log = logging.getLogger(__name__)
38
38
39
39
40 def create_git_wsgi_app(repo_path, repo_name, config):
40 def create_git_wsgi_app(repo_path, repo_name, config):
41 url = _vcs_streaming_url() + 'git/'
41 url = _vcs_streaming_url() + 'git/'
42 return VcsHttpProxy(url, repo_path, repo_name, config)
42 return VcsHttpProxy(url, repo_path, repo_name, config)
43
43
44
44
45 def create_hg_wsgi_app(repo_path, repo_name, config):
45 def create_hg_wsgi_app(repo_path, repo_name, config):
46 url = _vcs_streaming_url() + 'hg/'
46 url = _vcs_streaming_url() + 'hg/'
47 return VcsHttpProxy(url, repo_path, repo_name, config)
47 return VcsHttpProxy(url, repo_path, repo_name, config)
48
48
49
49
50 def _vcs_streaming_url():
50 def _vcs_streaming_url():
51 template = 'http://{}/stream/'
51 template = 'http://{}/stream/'
52 return template.format(rhodecode.CONFIG['vcs.server'])
52 return template.format(rhodecode.CONFIG['vcs.server'])
53
53
54
54
55 # TODO: johbo: Avoid the global.
55 # TODO: johbo: Avoid the global.
56 session = requests.Session()
56 session = requests.Session()
57 # Requests speedup, avoid reading .netrc and similar
57 # Requests speedup, avoid reading .netrc and similar
58 session.trust_env = False
58 session.trust_env = False
59
59
60 # prevent urllib3 spawning our logs.
60 # prevent urllib3 spawning our logs.
61 logging.getLogger("requests.packages.urllib3.connectionpool").setLevel(
61 logging.getLogger("requests.packages.urllib3.connectionpool").setLevel(
62 logging.WARNING)
62 logging.WARNING)
63
63
64
64
65 class VcsHttpProxy(object):
65 class VcsHttpProxy(object):
66 """
66 """
67 A WSGI application which proxies vcs requests.
67 A WSGI application which proxies vcs requests.
68
68
69 The goal is to shuffle the data around without touching it. The only
69 The goal is to shuffle the data around without touching it. The only
70 exception is the extra data from the config object which we send to the
70 exception is the extra data from the config object which we send to the
71 server as well.
71 server as well.
72 """
72 """
73
73
74 def __init__(self, url, repo_path, repo_name, config):
74 def __init__(self, url, repo_path, repo_name, config):
75 """
75 """
76 :param str url: The URL of the VCSServer to call.
76 :param str url: The URL of the VCSServer to call.
77 """
77 """
78 self._url = url
78 self._url = url
79 self._repo_name = repo_name
79 self._repo_name = repo_name
80 self._repo_path = repo_path
80 self._repo_path = repo_path
81 self._config = config
81 self._config = config
82 log.debug(
82 log.debug(
83 "Creating VcsHttpProxy for repo %s, url %s",
83 "Creating VcsHttpProxy for repo %s, url %s",
84 repo_name, url)
84 repo_name, url)
85
85
86 def __call__(self, environ, start_response):
86 def __call__(self, environ, start_response):
87 config = msgpack.packb(self._config)
87 config = msgpack.packb(self._config)
88 request = webob.request.Request(environ)
88 request = webob.request.Request(environ)
89 request_headers = request.headers
89 request_headers = request.headers
90 request_headers.update({
90 request_headers.update({
91 # TODO: johbo: Remove this, rely on URL path only
91 # TODO: johbo: Remove this, rely on URL path only
92 'X-RC-Repo-Name': self._repo_name,
92 'X-RC-Repo-Name': self._repo_name,
93 'X-RC-Repo-Path': self._repo_path,
93 'X-RC-Repo-Path': self._repo_path,
94 'X-RC-Path-Info': environ['PATH_INFO'],
94 'X-RC-Path-Info': environ['PATH_INFO'],
95 # TODO: johbo: Avoid encoding and put this into payload?
95 # TODO: johbo: Avoid encoding and put this into payload?
96 'X-RC-Repo-Config': base64.b64encode(config),
96 'X-RC-Repo-Config': base64.b64encode(config),
97 'X-RC-Locked-Status-Code': rhodecode.CONFIG.get('lock_ret_code')
97 'X-RC-Locked-Status-Code': rhodecode.CONFIG.get('lock_ret_code')
98 })
98 })
99
99
100 method = environ['REQUEST_METHOD']
100 method = environ['REQUEST_METHOD']
101
101
102 # Preserve the query string
102 # Preserve the query string
103 url = self._url
103 url = self._url
104 url = urlparse.urljoin(url, self._repo_name)
104 url = urlparse.urljoin(url, self._repo_name)
105 if environ.get('QUERY_STRING'):
105 if environ.get('QUERY_STRING'):
106 url += '?' + environ['QUERY_STRING']
106 url += '?' + environ['QUERY_STRING']
107
107
108 response = session.request(
108 response = session.request(
109 method, url,
109 method, url,
110 data=_maybe_stream_request(environ),
110 data=_maybe_stream_request(environ),
111 headers=request_headers,
111 headers=request_headers,
112 stream=True)
112 stream=True)
113
113
114 # Preserve the headers of the response, except hop_by_hop ones
114 # Preserve the headers of the response, except hop_by_hop ones
115 response_headers = [
115 response_headers = [
116 (h, v) for h, v in response.headers.items()
116 (h, v) for h, v in response.headers.items()
117 if not wsgiref.util.is_hop_by_hop(h)
117 if not wsgiref.util.is_hop_by_hop(h)
118 ]
118 ]
119
119
120 # Build status argument for start_reponse callable.
120 # Build status argument for start_reponse callable.
121 status = '{status_code} {reason_phrase}'.format(
121 status = '{status_code} {reason_phrase}'.format(
122 status_code=response.status_code,
122 status_code=response.status_code,
123 reason_phrase=response.reason)
123 reason_phrase=response.reason)
124
124
125 start_response(status, response_headers)
125 start_response(status, response_headers)
126 return _maybe_stream_response(response)
126 return _maybe_stream_response(response)
127
127
128
128
129 def _is_request_chunked(environ):
130 stream = environ.get('HTTP_TRANSFER_ENCODING', '') == 'chunked'
131 return stream
132
133
129 def _maybe_stream_request(environ):
134 def _maybe_stream_request(environ):
130 path = environ['PATH_INFO']
135 path = environ['PATH_INFO']
131 stream = _is_request_chunked(environ)
136 stream = _is_request_chunked(environ)
132 log.debug('handling request `%s` with stream support: %s', path, stream)
137 log.debug('handling request `%s` with stream support: %s', path, stream)
133
138
134 if stream:
139 if stream:
135 return environ['wsgi.input']
140 return environ['wsgi.input']
136 else:
141 else:
137 return environ['wsgi.input'].read()
142 return environ['wsgi.input'].read()
138
143
139
144
140 def _is_request_chunked(environ):
141 stream = environ.get('HTTP_TRANSFER_ENCODING', '') == 'chunked'
142 if not stream:
143 # git lfs should stream for PUT requests which are upload
144 stream = ('git-lfs' in environ.get('HTTP_USER_AGENT', '')
145 and environ['REQUEST_METHOD'] == 'PUT')
146 return stream
147
148
149 def _maybe_stream_response(response):
145 def _maybe_stream_response(response):
150 """
146 """
151 Try to generate chunks from the response if it is chunked.
147 Try to generate chunks from the response if it is chunked.
152 """
148 """
153 stream = _is_chunked(response)
149 stream = _is_chunked(response)
154 log.debug('returning response with stream: %s', stream)
150 log.debug('returning response with stream: %s', stream)
155 if stream:
151 if stream:
156 return response.raw.read_chunked()
152 return response.raw.read_chunked()
157 else:
153 else:
158 return [response.content]
154 return [response.content]
159
155
160
156
161 def _is_chunked(response):
157 def _is_chunked(response):
162 return response.headers.get('Transfer-Encoding', '') == 'chunked'
158 return response.headers.get('Transfer-Encoding', '') == 'chunked'
General Comments 0
You need to be logged in to leave comments. Login now