##// END OF EJS Templates
svn: use streaming uploads/downloads of files....
marcink -
r3022:211018a9 stable
parent child Browse files
Show More
@@ -1,204 +1,225 b''
1 1 # -*- coding: utf-8 -*-
2 2
3 3 # Copyright (C) 2010-2018 RhodeCode GmbH
4 4 #
5 5 # This program is free software: you can redistribute it and/or modify
6 6 # it under the terms of the GNU Affero General Public License, version 3
7 7 # (only), as published by the Free Software Foundation.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU Affero General Public License
15 15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 16 #
17 17 # This program is dual-licensed. If you wish to learn more about the
18 18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20 20
21 21 import base64
22 22 import logging
23 23 import urllib
24 from urlparse import urljoin
24 import urlparse
25 25
26 26 import requests
27 27 from pyramid.httpexceptions import HTTPNotAcceptable
28 28
29 29 from rhodecode.lib import rc_cache
30 30 from rhodecode.lib.middleware import simplevcs
31 31 from rhodecode.lib.utils import is_valid_repo
32 32 from rhodecode.lib.utils2 import str2bool, safe_int
33 33 from rhodecode.lib.ext_json import json
34 34 from rhodecode.lib.hooks_daemon import store_txn_id_data
35 35
36 36
37 37 log = logging.getLogger(__name__)
38 38
39 39
40 40 class SimpleSvnApp(object):
41 41 IGNORED_HEADERS = [
42 42 'connection', 'keep-alive', 'content-encoding',
43 43 'transfer-encoding', 'content-length']
44 44 rc_extras = {}
45 45
46 46 def __init__(self, config):
47 47 self.config = config
48 48
49 49 def __call__(self, environ, start_response):
50 50 request_headers = self._get_request_headers(environ)
51 data = environ['wsgi.input']
52 req_method = environ['REQUEST_METHOD']
53 has_content_length = 'CONTENT_LENGTH' in environ
54 path_info = self._get_url(environ['PATH_INFO'])
55 transfer_encoding = environ.get('HTTP_TRANSFER_ENCODING', '')
56 log.debug('Handling: %s method via `%s`', req_method, path_info)
51 57
52 data = environ['wsgi.input']
53 # johbo: Avoid that we end up with sending the request in chunked
54 # transfer encoding (mainly on Gunicorn). If we know the content
55 # length, then we should transfer the payload in one request.
56 if environ['REQUEST_METHOD'] == 'MKCOL' or 'CONTENT_LENGTH' in environ:
57 data = data.read()
58 if data.startswith('(create-txn-with-props'):
58 # stream control flag, based on request and content type...
59 stream = False
60
61 if req_method in ['MKCOL'] or has_content_length:
62 data_processed = False
63 # read chunk to check if we have txn-with-props
64 initial_data = data.read(1024)
65 if initial_data.startswith('(create-txn-with-props'):
66 data = initial_data + data.read()
59 67 # store on-the-fly our rc_extra using svn revision properties
60 68 # those can be read later on in hooks executed so we have a way
61 69 # to pass in the data into svn hooks
62 70 rc_data = base64.urlsafe_b64encode(json.dumps(self.rc_extras))
63 71 rc_data_len = len(rc_data)
64 # header defines data lenght, and serialized data
72 # header defines data length, and serialized data
65 73 skel = ' rc-scm-extras {} {}'.format(rc_data_len, rc_data)
66 74 data = data[:-2] + skel + '))'
75 data_processed = True
67 76
68 log.debug('Calling: %s method via `%s`', environ['REQUEST_METHOD'],
69 self._get_url(environ['PATH_INFO']))
77 if not data_processed:
78 # NOTE(johbo): Avoid that we end up with sending the request in chunked
79 # transfer encoding (mainly on Gunicorn). If we know the content
80 # length, then we should transfer the payload in one request.
81 data = initial_data + data.read()
70 82
83 if req_method in ['GET', 'PUT'] or transfer_encoding == 'chunked':
84 # NOTE(marcink): when getting/uploading files we want to STREAM content
85 # back to the client/proxy instead of buffering it here...
86 stream = True
87
88 stream = stream
89 log.debug(
90 'Calling SVN PROXY: method:%s via `%s`, Stream: %s',
91 req_method, path_info, stream)
71 92 response = requests.request(
72 environ['REQUEST_METHOD'], self._get_url(environ['PATH_INFO']),
73 data=data, headers=request_headers)
93 req_method, path_info,
94 data=data, headers=request_headers, stream=stream)
74 95
75 96 if response.status_code not in [200, 401]:
76 97 if response.status_code >= 500:
77 98 log.error('Got SVN response:%s with text:\n`%s`',
78 99 response, response.text)
79 100 else:
80 101 log.debug('Got SVN response:%s with text:\n`%s`',
81 102 response, response.text)
82 103 else:
83 104 log.debug('got response code: %s', response.status_code)
84 105
85 106 response_headers = self._get_response_headers(response.headers)
86 107
87 108 if response.headers.get('SVN-Txn-name'):
88 109 svn_tx_id = response.headers.get('SVN-Txn-name')
89 110 txn_id = rc_cache.utils.compute_key_from_params(
90 111 self.config['repository'], svn_tx_id)
91 112 port = safe_int(self.rc_extras['hooks_uri'].split(':')[-1])
92 113 store_txn_id_data(txn_id, {'port': port})
93 114
94 115 start_response(
95 116 '{} {}'.format(response.status_code, response.reason),
96 117 response_headers)
97 118 return response.iter_content(chunk_size=1024)
98 119
99 120 def _get_url(self, path):
100 url_path = urljoin(
121 url_path = urlparse.urljoin(
101 122 self.config.get('subversion_http_server_url', ''), path)
102 123 url_path = urllib.quote(url_path, safe="/:=~+!$,;'")
103 124 return url_path
104 125
105 126 def _get_request_headers(self, environ):
106 127 headers = {}
107 128
108 129 for key in environ:
109 130 if not key.startswith('HTTP_'):
110 131 continue
111 132 new_key = key.split('_')
112 133 new_key = [k.capitalize() for k in new_key[1:]]
113 134 new_key = '-'.join(new_key)
114 135 headers[new_key] = environ[key]
115 136
116 137 if 'CONTENT_TYPE' in environ:
117 138 headers['Content-Type'] = environ['CONTENT_TYPE']
118 139
119 140 if 'CONTENT_LENGTH' in environ:
120 141 headers['Content-Length'] = environ['CONTENT_LENGTH']
121 142
122 143 return headers
123 144
124 145 def _get_response_headers(self, headers):
125 146 headers = [
126 147 (h, headers[h])
127 148 for h in headers
128 149 if h.lower() not in self.IGNORED_HEADERS
129 150 ]
130 151
131 152 return headers
132 153
133 154
134 155 class DisabledSimpleSvnApp(object):
135 156 def __init__(self, config):
136 157 self.config = config
137 158
138 159 def __call__(self, environ, start_response):
139 160 reason = 'Cannot handle SVN call because: SVN HTTP Proxy is not enabled'
140 161 log.warning(reason)
141 162 return HTTPNotAcceptable(reason)(environ, start_response)
142 163
143 164
144 165 class SimpleSvn(simplevcs.SimpleVCS):
145 166
146 167 SCM = 'svn'
147 168 READ_ONLY_COMMANDS = ('OPTIONS', 'PROPFIND', 'GET', 'REPORT')
148 169 DEFAULT_HTTP_SERVER = 'http://localhost:8090'
149 170
150 171 def _get_repository_name(self, environ):
151 172 """
152 173 Gets repository name out of PATH_INFO header
153 174
154 175 :param environ: environ where PATH_INFO is stored
155 176 """
156 177 path = environ['PATH_INFO'].split('!')
157 178 repo_name = path[0].strip('/')
158 179
159 180 # SVN includes the whole path in it's requests, including
160 181 # subdirectories inside the repo. Therefore we have to search for
161 182 # the repo root directory.
162 183 if not is_valid_repo(
163 184 repo_name, self.base_path, explicit_scm=self.SCM):
164 185 current_path = ''
165 186 for component in repo_name.split('/'):
166 187 current_path += component
167 188 if is_valid_repo(
168 189 current_path, self.base_path, explicit_scm=self.SCM):
169 190 return current_path
170 191 current_path += '/'
171 192
172 193 return repo_name
173 194
174 195 def _get_action(self, environ):
175 196 return (
176 197 'pull'
177 198 if environ['REQUEST_METHOD'] in self.READ_ONLY_COMMANDS
178 199 else 'push')
179 200
180 201 def _should_use_callback_daemon(self, extras, environ, action):
181 202 # only MERGE command triggers hooks, so we don't want to start
182 203 # hooks server too many times. POST however starts the svn transaction
183 204 # so we also need to run the init of callback daemon of POST
184 205 if environ['REQUEST_METHOD'] in ['MERGE', 'POST']:
185 206 return True
186 207 return False
187 208
188 209 def _create_wsgi_app(self, repo_path, repo_name, config):
189 210 if self._is_svn_enabled():
190 211 return SimpleSvnApp(config)
191 212 # we don't have http proxy enabled return dummy request handler
192 213 return DisabledSimpleSvnApp(config)
193 214
194 215 def _is_svn_enabled(self):
195 216 conf = self.repo_vcs_config
196 217 return str2bool(conf.get('vcs_svn_proxy', 'http_requests_enabled'))
197 218
198 219 def _create_config(self, extras, repo_name):
199 220 conf = self.repo_vcs_config
200 221 server_url = conf.get('vcs_svn_proxy', 'http_server_url')
201 222 server_url = server_url or self.DEFAULT_HTTP_SERVER
202 223
203 224 extras['subversion_http_server_url'] = server_url
204 225 return extras
General Comments 0
You need to be logged in to leave comments. Login now