##// END OF EJS Templates
svn: use streaming uploads/downloads of files....
marcink -
r3056:869da457 default
parent child Browse files
Show More
@@ -1,204 +1,225 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2
2
3 # Copyright (C) 2010-2018 RhodeCode GmbH
3 # Copyright (C) 2010-2018 RhodeCode GmbH
4 #
4 #
5 # This program is free software: you can redistribute it and/or modify
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU Affero General Public License, version 3
6 # it under the terms of the GNU Affero General Public License, version 3
7 # (only), as published by the Free Software Foundation.
7 # (only), as published by the Free Software Foundation.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU Affero General Public License
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 #
16 #
17 # This program is dual-licensed. If you wish to learn more about the
17 # This program is dual-licensed. If you wish to learn more about the
18 # RhodeCode Enterprise Edition, including its added features, Support services,
18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20
20
21 import base64
21 import base64
22 import logging
22 import logging
23 import urllib
23 import urllib
24 from urlparse import urljoin
24 import urlparse
25
25
26 import requests
26 import requests
27 from pyramid.httpexceptions import HTTPNotAcceptable
27 from pyramid.httpexceptions import HTTPNotAcceptable
28
28
29 from rhodecode.lib import rc_cache
29 from rhodecode.lib import rc_cache
30 from rhodecode.lib.middleware import simplevcs
30 from rhodecode.lib.middleware import simplevcs
31 from rhodecode.lib.utils import is_valid_repo
31 from rhodecode.lib.utils import is_valid_repo
32 from rhodecode.lib.utils2 import str2bool, safe_int
32 from rhodecode.lib.utils2 import str2bool, safe_int
33 from rhodecode.lib.ext_json import json
33 from rhodecode.lib.ext_json import json
34 from rhodecode.lib.hooks_daemon import store_txn_id_data
34 from rhodecode.lib.hooks_daemon import store_txn_id_data
35
35
36
36
37 log = logging.getLogger(__name__)
37 log = logging.getLogger(__name__)
38
38
39
39
40 class SimpleSvnApp(object):
40 class SimpleSvnApp(object):
41 IGNORED_HEADERS = [
41 IGNORED_HEADERS = [
42 'connection', 'keep-alive', 'content-encoding',
42 'connection', 'keep-alive', 'content-encoding',
43 'transfer-encoding', 'content-length']
43 'transfer-encoding', 'content-length']
44 rc_extras = {}
44 rc_extras = {}
45
45
46 def __init__(self, config):
46 def __init__(self, config):
47 self.config = config
47 self.config = config
48
48
49 def __call__(self, environ, start_response):
49 def __call__(self, environ, start_response):
50 request_headers = self._get_request_headers(environ)
50 request_headers = self._get_request_headers(environ)
51 data = environ['wsgi.input']
52 req_method = environ['REQUEST_METHOD']
53 has_content_length = 'CONTENT_LENGTH' in environ
54 path_info = self._get_url(environ['PATH_INFO'])
55 transfer_encoding = environ.get('HTTP_TRANSFER_ENCODING', '')
56 log.debug('Handling: %s method via `%s`', req_method, path_info)
51
57
52 data = environ['wsgi.input']
58 # stream control flag, based on request and content type...
53 # johbo: Avoid that we end up with sending the request in chunked
59 stream = False
54 # transfer encoding (mainly on Gunicorn). If we know the content
60
55 # length, then we should transfer the payload in one request.
61 if req_method in ['MKCOL'] or has_content_length:
56 if environ['REQUEST_METHOD'] == 'MKCOL' or 'CONTENT_LENGTH' in environ:
62 data_processed = False
57 data = data.read()
63 # read chunk to check if we have txn-with-props
58 if data.startswith('(create-txn-with-props'):
64 initial_data = data.read(1024)
65 if initial_data.startswith('(create-txn-with-props'):
66 data = initial_data + data.read()
59 # store on-the-fly our rc_extra using svn revision properties
67 # store on-the-fly our rc_extra using svn revision properties
60 # those can be read later on in hooks executed so we have a way
68 # those can be read later on in hooks executed so we have a way
61 # to pass in the data into svn hooks
69 # to pass in the data into svn hooks
62 rc_data = base64.urlsafe_b64encode(json.dumps(self.rc_extras))
70 rc_data = base64.urlsafe_b64encode(json.dumps(self.rc_extras))
63 rc_data_len = len(rc_data)
71 rc_data_len = len(rc_data)
64 # header defines data lenght, and serialized data
72 # header defines data length, and serialized data
65 skel = ' rc-scm-extras {} {}'.format(rc_data_len, rc_data)
73 skel = ' rc-scm-extras {} {}'.format(rc_data_len, rc_data)
66 data = data[:-2] + skel + '))'
74 data = data[:-2] + skel + '))'
75 data_processed = True
67
76
68 log.debug('Calling: %s method via `%s`', environ['REQUEST_METHOD'],
77 if not data_processed:
69 self._get_url(environ['PATH_INFO']))
78 # NOTE(johbo): Avoid that we end up with sending the request in chunked
79 # transfer encoding (mainly on Gunicorn). If we know the content
80 # length, then we should transfer the payload in one request.
81 data = initial_data + data.read()
70
82
83 if req_method in ['GET', 'PUT'] or transfer_encoding == 'chunked':
84 # NOTE(marcink): when getting/uploading files we want to STREAM content
85 # back to the client/proxy instead of buffering it here...
86 stream = True
87
88 stream = stream
89 log.debug(
90 'Calling SVN PROXY: method:%s via `%s`, Stream: %s',
91 req_method, path_info, stream)
71 response = requests.request(
92 response = requests.request(
72 environ['REQUEST_METHOD'], self._get_url(environ['PATH_INFO']),
93 req_method, path_info,
73 data=data, headers=request_headers)
94 data=data, headers=request_headers, stream=stream)
74
95
75 if response.status_code not in [200, 401]:
96 if response.status_code not in [200, 401]:
76 if response.status_code >= 500:
97 if response.status_code >= 500:
77 log.error('Got SVN response:%s with text:\n`%s`',
98 log.error('Got SVN response:%s with text:\n`%s`',
78 response, response.text)
99 response, response.text)
79 else:
100 else:
80 log.debug('Got SVN response:%s with text:\n`%s`',
101 log.debug('Got SVN response:%s with text:\n`%s`',
81 response, response.text)
102 response, response.text)
82 else:
103 else:
83 log.debug('got response code: %s', response.status_code)
104 log.debug('got response code: %s', response.status_code)
84
105
85 response_headers = self._get_response_headers(response.headers)
106 response_headers = self._get_response_headers(response.headers)
86
107
87 if response.headers.get('SVN-Txn-name'):
108 if response.headers.get('SVN-Txn-name'):
88 svn_tx_id = response.headers.get('SVN-Txn-name')
109 svn_tx_id = response.headers.get('SVN-Txn-name')
89 txn_id = rc_cache.utils.compute_key_from_params(
110 txn_id = rc_cache.utils.compute_key_from_params(
90 self.config['repository'], svn_tx_id)
111 self.config['repository'], svn_tx_id)
91 port = safe_int(self.rc_extras['hooks_uri'].split(':')[-1])
112 port = safe_int(self.rc_extras['hooks_uri'].split(':')[-1])
92 store_txn_id_data(txn_id, {'port': port})
113 store_txn_id_data(txn_id, {'port': port})
93
114
94 start_response(
115 start_response(
95 '{} {}'.format(response.status_code, response.reason),
116 '{} {}'.format(response.status_code, response.reason),
96 response_headers)
117 response_headers)
97 return response.iter_content(chunk_size=1024)
118 return response.iter_content(chunk_size=1024)
98
119
99 def _get_url(self, path):
120 def _get_url(self, path):
100 url_path = urljoin(
121 url_path = urlparse.urljoin(
101 self.config.get('subversion_http_server_url', ''), path)
122 self.config.get('subversion_http_server_url', ''), path)
102 url_path = urllib.quote(url_path, safe="/:=~+!$,;'")
123 url_path = urllib.quote(url_path, safe="/:=~+!$,;'")
103 return url_path
124 return url_path
104
125
105 def _get_request_headers(self, environ):
126 def _get_request_headers(self, environ):
106 headers = {}
127 headers = {}
107
128
108 for key in environ:
129 for key in environ:
109 if not key.startswith('HTTP_'):
130 if not key.startswith('HTTP_'):
110 continue
131 continue
111 new_key = key.split('_')
132 new_key = key.split('_')
112 new_key = [k.capitalize() for k in new_key[1:]]
133 new_key = [k.capitalize() for k in new_key[1:]]
113 new_key = '-'.join(new_key)
134 new_key = '-'.join(new_key)
114 headers[new_key] = environ[key]
135 headers[new_key] = environ[key]
115
136
116 if 'CONTENT_TYPE' in environ:
137 if 'CONTENT_TYPE' in environ:
117 headers['Content-Type'] = environ['CONTENT_TYPE']
138 headers['Content-Type'] = environ['CONTENT_TYPE']
118
139
119 if 'CONTENT_LENGTH' in environ:
140 if 'CONTENT_LENGTH' in environ:
120 headers['Content-Length'] = environ['CONTENT_LENGTH']
141 headers['Content-Length'] = environ['CONTENT_LENGTH']
121
142
122 return headers
143 return headers
123
144
124 def _get_response_headers(self, headers):
145 def _get_response_headers(self, headers):
125 headers = [
146 headers = [
126 (h, headers[h])
147 (h, headers[h])
127 for h in headers
148 for h in headers
128 if h.lower() not in self.IGNORED_HEADERS
149 if h.lower() not in self.IGNORED_HEADERS
129 ]
150 ]
130
151
131 return headers
152 return headers
132
153
133
154
134 class DisabledSimpleSvnApp(object):
155 class DisabledSimpleSvnApp(object):
135 def __init__(self, config):
156 def __init__(self, config):
136 self.config = config
157 self.config = config
137
158
138 def __call__(self, environ, start_response):
159 def __call__(self, environ, start_response):
139 reason = 'Cannot handle SVN call because: SVN HTTP Proxy is not enabled'
160 reason = 'Cannot handle SVN call because: SVN HTTP Proxy is not enabled'
140 log.warning(reason)
161 log.warning(reason)
141 return HTTPNotAcceptable(reason)(environ, start_response)
162 return HTTPNotAcceptable(reason)(environ, start_response)
142
163
143
164
144 class SimpleSvn(simplevcs.SimpleVCS):
165 class SimpleSvn(simplevcs.SimpleVCS):
145
166
146 SCM = 'svn'
167 SCM = 'svn'
147 READ_ONLY_COMMANDS = ('OPTIONS', 'PROPFIND', 'GET', 'REPORT')
168 READ_ONLY_COMMANDS = ('OPTIONS', 'PROPFIND', 'GET', 'REPORT')
148 DEFAULT_HTTP_SERVER = 'http://localhost:8090'
169 DEFAULT_HTTP_SERVER = 'http://localhost:8090'
149
170
150 def _get_repository_name(self, environ):
171 def _get_repository_name(self, environ):
151 """
172 """
152 Gets repository name out of PATH_INFO header
173 Gets repository name out of PATH_INFO header
153
174
154 :param environ: environ where PATH_INFO is stored
175 :param environ: environ where PATH_INFO is stored
155 """
176 """
156 path = environ['PATH_INFO'].split('!')
177 path = environ['PATH_INFO'].split('!')
157 repo_name = path[0].strip('/')
178 repo_name = path[0].strip('/')
158
179
159 # SVN includes the whole path in it's requests, including
180 # SVN includes the whole path in it's requests, including
160 # subdirectories inside the repo. Therefore we have to search for
181 # subdirectories inside the repo. Therefore we have to search for
161 # the repo root directory.
182 # the repo root directory.
162 if not is_valid_repo(
183 if not is_valid_repo(
163 repo_name, self.base_path, explicit_scm=self.SCM):
184 repo_name, self.base_path, explicit_scm=self.SCM):
164 current_path = ''
185 current_path = ''
165 for component in repo_name.split('/'):
186 for component in repo_name.split('/'):
166 current_path += component
187 current_path += component
167 if is_valid_repo(
188 if is_valid_repo(
168 current_path, self.base_path, explicit_scm=self.SCM):
189 current_path, self.base_path, explicit_scm=self.SCM):
169 return current_path
190 return current_path
170 current_path += '/'
191 current_path += '/'
171
192
172 return repo_name
193 return repo_name
173
194
174 def _get_action(self, environ):
195 def _get_action(self, environ):
175 return (
196 return (
176 'pull'
197 'pull'
177 if environ['REQUEST_METHOD'] in self.READ_ONLY_COMMANDS
198 if environ['REQUEST_METHOD'] in self.READ_ONLY_COMMANDS
178 else 'push')
199 else 'push')
179
200
180 def _should_use_callback_daemon(self, extras, environ, action):
201 def _should_use_callback_daemon(self, extras, environ, action):
181 # only MERGE command triggers hooks, so we don't want to start
202 # only MERGE command triggers hooks, so we don't want to start
182 # hooks server too many times. POST however starts the svn transaction
203 # hooks server too many times. POST however starts the svn transaction
183 # so we also need to run the init of callback daemon of POST
204 # so we also need to run the init of callback daemon of POST
184 if environ['REQUEST_METHOD'] in ['MERGE', 'POST']:
205 if environ['REQUEST_METHOD'] in ['MERGE', 'POST']:
185 return True
206 return True
186 return False
207 return False
187
208
188 def _create_wsgi_app(self, repo_path, repo_name, config):
209 def _create_wsgi_app(self, repo_path, repo_name, config):
189 if self._is_svn_enabled():
210 if self._is_svn_enabled():
190 return SimpleSvnApp(config)
211 return SimpleSvnApp(config)
191 # we don't have http proxy enabled return dummy request handler
212 # we don't have http proxy enabled return dummy request handler
192 return DisabledSimpleSvnApp(config)
213 return DisabledSimpleSvnApp(config)
193
214
194 def _is_svn_enabled(self):
215 def _is_svn_enabled(self):
195 conf = self.repo_vcs_config
216 conf = self.repo_vcs_config
196 return str2bool(conf.get('vcs_svn_proxy', 'http_requests_enabled'))
217 return str2bool(conf.get('vcs_svn_proxy', 'http_requests_enabled'))
197
218
198 def _create_config(self, extras, repo_name):
219 def _create_config(self, extras, repo_name):
199 conf = self.repo_vcs_config
220 conf = self.repo_vcs_config
200 server_url = conf.get('vcs_svn_proxy', 'http_server_url')
221 server_url = conf.get('vcs_svn_proxy', 'http_server_url')
201 server_url = server_url or self.DEFAULT_HTTP_SERVER
222 server_url = server_url or self.DEFAULT_HTTP_SERVER
202
223
203 extras['subversion_http_server_url'] = server_url
224 extras['subversion_http_server_url'] = server_url
204 return extras
225 return extras
General Comments 0
You need to be logged in to leave comments. Login now