##// END OF EJS Templates
svn: add better connection error logging in case the SVN backend is offline
marcink -
r3573:bcafaf1e default
parent child Browse files
Show More
@@ -1,226 +1,228 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2
2
3 # Copyright (C) 2010-2019 RhodeCode GmbH
3 # Copyright (C) 2010-2019 RhodeCode GmbH
4 #
4 #
5 # This program is free software: you can redistribute it and/or modify
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU Affero General Public License, version 3
6 # it under the terms of the GNU Affero General Public License, version 3
7 # (only), as published by the Free Software Foundation.
7 # (only), as published by the Free Software Foundation.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU Affero General Public License
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 #
16 #
17 # This program is dual-licensed. If you wish to learn more about the
17 # This program is dual-licensed. If you wish to learn more about the
18 # RhodeCode Enterprise Edition, including its added features, Support services,
18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20
20
21 import base64
21 import base64
22 import logging
22 import logging
23 import urllib
23 import urllib
24 import urlparse
24 import urlparse
25
25
26 import requests
26 import requests
27 from pyramid.httpexceptions import HTTPNotAcceptable
27 from pyramid.httpexceptions import HTTPNotAcceptable
28
28
29 from rhodecode.lib import rc_cache
29 from rhodecode.lib import rc_cache
30 from rhodecode.lib.middleware import simplevcs
30 from rhodecode.lib.middleware import simplevcs
31 from rhodecode.lib.utils import is_valid_repo
31 from rhodecode.lib.utils import is_valid_repo
32 from rhodecode.lib.utils2 import str2bool, safe_int
32 from rhodecode.lib.utils2 import str2bool, safe_int
33 from rhodecode.lib.ext_json import json
33 from rhodecode.lib.ext_json import json
34 from rhodecode.lib.hooks_daemon import store_txn_id_data
34 from rhodecode.lib.hooks_daemon import store_txn_id_data
35
35
36
36
37 log = logging.getLogger(__name__)
37 log = logging.getLogger(__name__)
38
38
39
39
40 class SimpleSvnApp(object):
40 class SimpleSvnApp(object):
41 IGNORED_HEADERS = [
41 IGNORED_HEADERS = [
42 'connection', 'keep-alive', 'content-encoding',
42 'connection', 'keep-alive', 'content-encoding',
43 'transfer-encoding', 'content-length']
43 'transfer-encoding', 'content-length']
44 rc_extras = {}
44 rc_extras = {}
45
45
46 def __init__(self, config):
46 def __init__(self, config):
47 self.config = config
47 self.config = config
48
48
49 def __call__(self, environ, start_response):
49 def __call__(self, environ, start_response):
50 request_headers = self._get_request_headers(environ)
50 request_headers = self._get_request_headers(environ)
51 data = environ['wsgi.input']
51 data = environ['wsgi.input']
52 req_method = environ['REQUEST_METHOD']
52 req_method = environ['REQUEST_METHOD']
53 has_content_length = 'CONTENT_LENGTH' in environ
53 has_content_length = 'CONTENT_LENGTH' in environ
54 path_info = self._get_url(
54 path_info = self._get_url(
55 self.config.get('subversion_http_server_url', ''), environ['PATH_INFO'])
55 self.config.get('subversion_http_server_url', ''), environ['PATH_INFO'])
56 transfer_encoding = environ.get('HTTP_TRANSFER_ENCODING', '')
56 transfer_encoding = environ.get('HTTP_TRANSFER_ENCODING', '')
57 log.debug('Handling: %s method via `%s`', req_method, path_info)
57 log.debug('Handling: %s method via `%s`', req_method, path_info)
58
58
59 # stream control flag, based on request and content type...
59 # stream control flag, based on request and content type...
60 stream = False
60 stream = False
61
61
62 if req_method in ['MKCOL'] or has_content_length:
62 if req_method in ['MKCOL'] or has_content_length:
63 data_processed = False
63 data_processed = False
64 # read chunk to check if we have txn-with-props
64 # read chunk to check if we have txn-with-props
65 initial_data = data.read(1024)
65 initial_data = data.read(1024)
66 if initial_data.startswith('(create-txn-with-props'):
66 if initial_data.startswith('(create-txn-with-props'):
67 data = initial_data + data.read()
67 data = initial_data + data.read()
68 # store on-the-fly our rc_extra using svn revision properties
68 # store on-the-fly our rc_extra using svn revision properties
69 # those can be read later on in hooks executed so we have a way
69 # those can be read later on in hooks executed so we have a way
70 # to pass in the data into svn hooks
70 # to pass in the data into svn hooks
71 rc_data = base64.urlsafe_b64encode(json.dumps(self.rc_extras))
71 rc_data = base64.urlsafe_b64encode(json.dumps(self.rc_extras))
72 rc_data_len = len(rc_data)
72 rc_data_len = len(rc_data)
73 # header defines data length, and serialized data
73 # header defines data length, and serialized data
74 skel = ' rc-scm-extras {} {}'.format(rc_data_len, rc_data)
74 skel = ' rc-scm-extras {} {}'.format(rc_data_len, rc_data)
75 data = data[:-2] + skel + '))'
75 data = data[:-2] + skel + '))'
76 data_processed = True
76 data_processed = True
77
77
78 if not data_processed:
78 if not data_processed:
79 # NOTE(johbo): Avoid that we end up with sending the request in chunked
79 # NOTE(johbo): Avoid that we end up with sending the request in chunked
80 # transfer encoding (mainly on Gunicorn). If we know the content
80 # transfer encoding (mainly on Gunicorn). If we know the content
81 # length, then we should transfer the payload in one request.
81 # length, then we should transfer the payload in one request.
82 data = initial_data + data.read()
82 data = initial_data + data.read()
83
83
84 if req_method in ['GET', 'PUT'] or transfer_encoding == 'chunked':
84 if req_method in ['GET', 'PUT'] or transfer_encoding == 'chunked':
85 # NOTE(marcink): when getting/uploading files we want to STREAM content
85 # NOTE(marcink): when getting/uploading files we want to STREAM content
86 # back to the client/proxy instead of buffering it here...
86 # back to the client/proxy instead of buffering it here...
87 stream = True
87 stream = True
88
88
89 stream = stream
89 stream = stream
90 log.debug(
90 log.debug('Calling SVN PROXY at `%s`, using method:%s. Stream: %s',
91 'Calling SVN PROXY: method:%s via `%s`, Stream: %s',
91 path_info, req_method, stream)
92 req_method, path_info, stream)
92 try:
93 response = requests.request(
93 response = requests.request(
94 req_method, path_info,
94 req_method, path_info,
95 data=data, headers=request_headers, stream=stream)
95 data=data, headers=request_headers, stream=stream)
96 except requests.ConnectionError:
97 log.exception('ConnectionError occurred for endpoint %s', path_info)
98 raise
96
99
97 if response.status_code not in [200, 401]:
100 if response.status_code not in [200, 401]:
101 text = '\n{}'.format(response.text) if response.text else ''
98 if response.status_code >= 500:
102 if response.status_code >= 500:
99 log.error('Got SVN response:%s with text:\n`%s`',
103 log.error('Got SVN response:%s with text:`%s`', response, text)
100 response, response.text)
101 else:
104 else:
102 log.debug('Got SVN response:%s with text:\n`%s`',
105 log.debug('Got SVN response:%s with text:`%s`', response, text)
103 response, response.text)
104 else:
106 else:
105 log.debug('got response code: %s', response.status_code)
107 log.debug('got response code: %s', response.status_code)
106
108
107 response_headers = self._get_response_headers(response.headers)
109 response_headers = self._get_response_headers(response.headers)
108
110
109 if response.headers.get('SVN-Txn-name'):
111 if response.headers.get('SVN-Txn-name'):
110 svn_tx_id = response.headers.get('SVN-Txn-name')
112 svn_tx_id = response.headers.get('SVN-Txn-name')
111 txn_id = rc_cache.utils.compute_key_from_params(
113 txn_id = rc_cache.utils.compute_key_from_params(
112 self.config['repository'], svn_tx_id)
114 self.config['repository'], svn_tx_id)
113 port = safe_int(self.rc_extras['hooks_uri'].split(':')[-1])
115 port = safe_int(self.rc_extras['hooks_uri'].split(':')[-1])
114 store_txn_id_data(txn_id, {'port': port})
116 store_txn_id_data(txn_id, {'port': port})
115
117
116 start_response(
118 start_response(
117 '{} {}'.format(response.status_code, response.reason),
119 '{} {}'.format(response.status_code, response.reason),
118 response_headers)
120 response_headers)
119 return response.iter_content(chunk_size=1024)
121 return response.iter_content(chunk_size=1024)
120
122
121 def _get_url(self, svn_http_server, path):
123 def _get_url(self, svn_http_server, path):
122 svn_http_server_url = (svn_http_server or '').rstrip('/')
124 svn_http_server_url = (svn_http_server or '').rstrip('/')
123 url_path = urlparse.urljoin(svn_http_server_url + '/', (path or '').lstrip('/'))
125 url_path = urlparse.urljoin(svn_http_server_url + '/', (path or '').lstrip('/'))
124 url_path = urllib.quote(url_path, safe="/:=~+!$,;'")
126 url_path = urllib.quote(url_path, safe="/:=~+!$,;'")
125 return url_path
127 return url_path
126
128
127 def _get_request_headers(self, environ):
129 def _get_request_headers(self, environ):
128 headers = {}
130 headers = {}
129
131
130 for key in environ:
132 for key in environ:
131 if not key.startswith('HTTP_'):
133 if not key.startswith('HTTP_'):
132 continue
134 continue
133 new_key = key.split('_')
135 new_key = key.split('_')
134 new_key = [k.capitalize() for k in new_key[1:]]
136 new_key = [k.capitalize() for k in new_key[1:]]
135 new_key = '-'.join(new_key)
137 new_key = '-'.join(new_key)
136 headers[new_key] = environ[key]
138 headers[new_key] = environ[key]
137
139
138 if 'CONTENT_TYPE' in environ:
140 if 'CONTENT_TYPE' in environ:
139 headers['Content-Type'] = environ['CONTENT_TYPE']
141 headers['Content-Type'] = environ['CONTENT_TYPE']
140
142
141 if 'CONTENT_LENGTH' in environ:
143 if 'CONTENT_LENGTH' in environ:
142 headers['Content-Length'] = environ['CONTENT_LENGTH']
144 headers['Content-Length'] = environ['CONTENT_LENGTH']
143
145
144 return headers
146 return headers
145
147
146 def _get_response_headers(self, headers):
148 def _get_response_headers(self, headers):
147 headers = [
149 headers = [
148 (h, headers[h])
150 (h, headers[h])
149 for h in headers
151 for h in headers
150 if h.lower() not in self.IGNORED_HEADERS
152 if h.lower() not in self.IGNORED_HEADERS
151 ]
153 ]
152
154
153 return headers
155 return headers
154
156
155
157
156 class DisabledSimpleSvnApp(object):
158 class DisabledSimpleSvnApp(object):
157 def __init__(self, config):
159 def __init__(self, config):
158 self.config = config
160 self.config = config
159
161
160 def __call__(self, environ, start_response):
162 def __call__(self, environ, start_response):
161 reason = 'Cannot handle SVN call because: SVN HTTP Proxy is not enabled'
163 reason = 'Cannot handle SVN call because: SVN HTTP Proxy is not enabled'
162 log.warning(reason)
164 log.warning(reason)
163 return HTTPNotAcceptable(reason)(environ, start_response)
165 return HTTPNotAcceptable(reason)(environ, start_response)
164
166
165
167
166 class SimpleSvn(simplevcs.SimpleVCS):
168 class SimpleSvn(simplevcs.SimpleVCS):
167
169
168 SCM = 'svn'
170 SCM = 'svn'
169 READ_ONLY_COMMANDS = ('OPTIONS', 'PROPFIND', 'GET', 'REPORT')
171 READ_ONLY_COMMANDS = ('OPTIONS', 'PROPFIND', 'GET', 'REPORT')
170 DEFAULT_HTTP_SERVER = 'http://localhost:8090'
172 DEFAULT_HTTP_SERVER = 'http://localhost:8090'
171
173
172 def _get_repository_name(self, environ):
174 def _get_repository_name(self, environ):
173 """
175 """
174 Gets repository name out of PATH_INFO header
176 Gets repository name out of PATH_INFO header
175
177
176 :param environ: environ where PATH_INFO is stored
178 :param environ: environ where PATH_INFO is stored
177 """
179 """
178 path = environ['PATH_INFO'].split('!')
180 path = environ['PATH_INFO'].split('!')
179 repo_name = path[0].strip('/')
181 repo_name = path[0].strip('/')
180
182
181 # SVN includes the whole path in it's requests, including
183 # SVN includes the whole path in it's requests, including
182 # subdirectories inside the repo. Therefore we have to search for
184 # subdirectories inside the repo. Therefore we have to search for
183 # the repo root directory.
185 # the repo root directory.
184 if not is_valid_repo(
186 if not is_valid_repo(
185 repo_name, self.base_path, explicit_scm=self.SCM):
187 repo_name, self.base_path, explicit_scm=self.SCM):
186 current_path = ''
188 current_path = ''
187 for component in repo_name.split('/'):
189 for component in repo_name.split('/'):
188 current_path += component
190 current_path += component
189 if is_valid_repo(
191 if is_valid_repo(
190 current_path, self.base_path, explicit_scm=self.SCM):
192 current_path, self.base_path, explicit_scm=self.SCM):
191 return current_path
193 return current_path
192 current_path += '/'
194 current_path += '/'
193
195
194 return repo_name
196 return repo_name
195
197
196 def _get_action(self, environ):
198 def _get_action(self, environ):
197 return (
199 return (
198 'pull'
200 'pull'
199 if environ['REQUEST_METHOD'] in self.READ_ONLY_COMMANDS
201 if environ['REQUEST_METHOD'] in self.READ_ONLY_COMMANDS
200 else 'push')
202 else 'push')
201
203
202 def _should_use_callback_daemon(self, extras, environ, action):
204 def _should_use_callback_daemon(self, extras, environ, action):
203 # only MERGE command triggers hooks, so we don't want to start
205 # only MERGE command triggers hooks, so we don't want to start
204 # hooks server too many times. POST however starts the svn transaction
206 # hooks server too many times. POST however starts the svn transaction
205 # so we also need to run the init of callback daemon of POST
207 # so we also need to run the init of callback daemon of POST
206 if environ['REQUEST_METHOD'] in ['MERGE', 'POST']:
208 if environ['REQUEST_METHOD'] in ['MERGE', 'POST']:
207 return True
209 return True
208 return False
210 return False
209
211
210 def _create_wsgi_app(self, repo_path, repo_name, config):
212 def _create_wsgi_app(self, repo_path, repo_name, config):
211 if self._is_svn_enabled():
213 if self._is_svn_enabled():
212 return SimpleSvnApp(config)
214 return SimpleSvnApp(config)
213 # we don't have http proxy enabled return dummy request handler
215 # we don't have http proxy enabled return dummy request handler
214 return DisabledSimpleSvnApp(config)
216 return DisabledSimpleSvnApp(config)
215
217
216 def _is_svn_enabled(self):
218 def _is_svn_enabled(self):
217 conf = self.repo_vcs_config
219 conf = self.repo_vcs_config
218 return str2bool(conf.get('vcs_svn_proxy', 'http_requests_enabled'))
220 return str2bool(conf.get('vcs_svn_proxy', 'http_requests_enabled'))
219
221
220 def _create_config(self, extras, repo_name):
222 def _create_config(self, extras, repo_name):
221 conf = self.repo_vcs_config
223 conf = self.repo_vcs_config
222 server_url = conf.get('vcs_svn_proxy', 'http_server_url')
224 server_url = conf.get('vcs_svn_proxy', 'http_server_url')
223 server_url = server_url or self.DEFAULT_HTTP_SERVER
225 server_url = server_url or self.DEFAULT_HTTP_SERVER
224
226
225 extras['subversion_http_server_url'] = server_url
227 extras['subversion_http_server_url'] = server_url
226 return extras
228 return extras
General Comments 0
You need to be logged in to leave comments. Login now