##// END OF EJS Templates
core: use application wide detection of invalid bytes sent via URL/GET/POST data.
marcink -
r3145:f9f61897 default
parent child Browse files
Show More
@@ -0,0 +1,45 b''
1 # -*- coding: utf-8 -*-
2
3 # Copyright (C) 2016-2018 RhodeCode GmbH
4 #
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU Affero General Public License, version 3
7 # (only), as published by the Free Software Foundation.
8 #
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
13 #
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 #
17 # This program is dual-licensed. If you wish to learn more about the
18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20
21 import pytest
22
23 from rhodecode.tests import TestController
24 from rhodecode.tests.fixture import Fixture
25
26
27 class TestBadRequestData(TestController):
28
29 def test_bad_get_data(self):
30 self.app.get(
31 '/', params={'f\xfc': '\xfc%f6%22%20onmouseover%3dveA2(9352)%20'},
32 status=400)
33
34 def test_bad_url_data(self):
35 self.app.post(
36 '/f\xfc',
37 status=400)
38
39 def test_bad_post_data(self, csrf_token, xhr_header):
40 self.app.post(
41 '/_markup_preview',
42 params={'f\xfc': '\xfc%f6%22%20onmouseover%3dveA2(9352)%20',
43 'csrf_token': csrf_token},
44 extra_environ=xhr_header,
45 status=400)
@@ -1,341 +1,327 b''
1 1 # -*- coding: utf-8 -*-
2 2
3 3 # Copyright (C) 2010-2018 RhodeCode GmbH
4 4 #
5 5 # This program is free software: you can redistribute it and/or modify
6 6 # it under the terms of the GNU Affero General Public License, version 3
7 7 # (only), as published by the Free Software Foundation.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU Affero General Public License
15 15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 16 #
17 17 # This program is dual-licensed. If you wish to learn more about the
18 18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20 20 import io
21 21 import re
22 22 import os
23 23 import datetime
24 24 import logging
25 25 import Queue
26 26 import subprocess32
27 27
28 28
29 29 from dateutil.parser import parse
30 from pyramid.httpexceptions import HTTPBadRequest
31 30 from pyramid.threadlocal import get_current_request
32 31 from pyramid.interfaces import IRoutesMapper
33 32 from pyramid.settings import asbool
34 33 from pyramid.path import AssetResolver
35 34 from threading import Thread
36 35
37 36 from rhodecode.translation import _ as tsf
38 37 from rhodecode.config.jsroutes import generate_jsroutes_content
39 38 from rhodecode.lib import auth
40 39 from rhodecode.lib.base import get_auth_user
41 40
42
43 41 import rhodecode
44 42
45 43
46 44 log = logging.getLogger(__name__)
47 45
48 46
49 47 def add_renderer_globals(event):
50 48 from rhodecode.lib import helpers
51 49
52 50 # TODO: When executed in pyramid view context the request is not available
53 51 # in the event. Find a better solution to get the request.
54 52 request = event['request'] or get_current_request()
55 53
56 54 # Add Pyramid translation as '_' to context
57 55 event['_'] = request.translate
58 56 event['_ungettext'] = request.plularize
59 57 event['h'] = helpers
60 58
61 59
62 60 def add_localizer(event):
63 61 request = event.request
64 try:
65 62 localizer = request.localizer
66 except Exception:
67 log.exception('Failed to get localizer')
68 # NOTE(marcink): Handle bug in pyramid by malformed GET params could crash
69 # this resulting on various odd errors on missing translators
70 # see: https://github.com/Pylons/pyramid/issues/3399
71
72 def dummy_translate(*args, **kwargs):
73 return ''.join(args)
74 request.translate = tsf
75 request.plularize = dummy_translate
76 raise HTTPBadRequest()
77 63
78 64 def auto_translate(*args, **kwargs):
79 65 return localizer.translate(tsf(*args, **kwargs))
80 66
81 67 request.translate = auto_translate
82 68 request.plularize = localizer.pluralize
83 69
84 70
85 71 def set_user_lang(event):
86 72 request = event.request
87 73 cur_user = getattr(request, 'user', None)
88 74
89 75 if cur_user:
90 76 user_lang = cur_user.get_instance().user_data.get('language')
91 77 if user_lang:
92 78 log.debug('lang: setting current user:%s language to: %s', cur_user, user_lang)
93 79 event.request._LOCALE_ = user_lang
94 80
95 81
96 82 def add_request_user_context(event):
97 83 """
98 84 Adds auth user into request context
99 85 """
100 86 request = event.request
101 87 # access req_id as soon as possible
102 88 req_id = request.req_id
103 89
104 90 if hasattr(request, 'vcs_call'):
105 91 # skip vcs calls
106 92 return
107 93
108 94 if hasattr(request, 'rpc_method'):
109 95 # skip api calls
110 96 return
111 97
112 98 auth_user = get_auth_user(request)
113 99 request.user = auth_user
114 100 request.environ['rc_auth_user'] = auth_user
115 101 request.environ['rc_auth_user_id'] = auth_user.user_id
116 102 request.environ['rc_req_id'] = req_id
117 103
118 104
119 105 def inject_app_settings(event):
120 106 settings = event.app.registry.settings
121 107 # inject info about available permissions
122 108 auth.set_available_permissions(settings)
123 109
124 110
125 111 def scan_repositories_if_enabled(event):
126 112 """
127 113 This is subscribed to the `pyramid.events.ApplicationCreated` event. It
128 114 does a repository scan if enabled in the settings.
129 115 """
130 116 settings = event.app.registry.settings
131 117 vcs_server_enabled = settings['vcs.server.enable']
132 118 import_on_startup = settings['startup.import_repos']
133 119 if vcs_server_enabled and import_on_startup:
134 120 from rhodecode.model.scm import ScmModel
135 121 from rhodecode.lib.utils import repo2db_mapper, get_rhodecode_base_path
136 122 repositories = ScmModel().repo_scan(get_rhodecode_base_path())
137 123 repo2db_mapper(repositories, remove_obsolete=False)
138 124
139 125
140 126 def write_metadata_if_needed(event):
141 127 """
142 128 Writes upgrade metadata
143 129 """
144 130 import rhodecode
145 131 from rhodecode.lib import system_info
146 132 from rhodecode.lib import ext_json
147 133
148 134 fname = '.rcmetadata.json'
149 135 ini_loc = os.path.dirname(rhodecode.CONFIG.get('__file__'))
150 136 metadata_destination = os.path.join(ini_loc, fname)
151 137
152 138 def get_update_age():
153 139 now = datetime.datetime.utcnow()
154 140
155 141 with open(metadata_destination, 'rb') as f:
156 142 data = ext_json.json.loads(f.read())
157 143 if 'created_on' in data:
158 144 update_date = parse(data['created_on'])
159 145 diff = now - update_date
160 146 return diff.total_seconds() / 60.0
161 147
162 148 return 0
163 149
164 150 def write():
165 151 configuration = system_info.SysInfo(
166 152 system_info.rhodecode_config)()['value']
167 153 license_token = configuration['config']['license_token']
168 154
169 155 setup = dict(
170 156 workers=configuration['config']['server:main'].get(
171 157 'workers', '?'),
172 158 worker_type=configuration['config']['server:main'].get(
173 159 'worker_class', 'sync'),
174 160 )
175 161 dbinfo = system_info.SysInfo(system_info.database_info)()['value']
176 162 del dbinfo['url']
177 163
178 164 metadata = dict(
179 165 desc='upgrade metadata info',
180 166 license_token=license_token,
181 167 created_on=datetime.datetime.utcnow().isoformat(),
182 168 usage=system_info.SysInfo(system_info.usage_info)()['value'],
183 169 platform=system_info.SysInfo(system_info.platform_type)()['value'],
184 170 database=dbinfo,
185 171 cpu=system_info.SysInfo(system_info.cpu)()['value'],
186 172 memory=system_info.SysInfo(system_info.memory)()['value'],
187 173 setup=setup
188 174 )
189 175
190 176 with open(metadata_destination, 'wb') as f:
191 177 f.write(ext_json.json.dumps(metadata))
192 178
193 179 settings = event.app.registry.settings
194 180 if settings.get('metadata.skip'):
195 181 return
196 182
197 183 # only write this every 24h, workers restart caused unwanted delays
198 184 try:
199 185 age_in_min = get_update_age()
200 186 except Exception:
201 187 age_in_min = 0
202 188
203 189 if age_in_min > 60 * 60 * 24:
204 190 return
205 191
206 192 try:
207 193 write()
208 194 except Exception:
209 195 pass
210 196
211 197
212 198 def write_js_routes_if_enabled(event):
213 199 registry = event.app.registry
214 200
215 201 mapper = registry.queryUtility(IRoutesMapper)
216 202 _argument_prog = re.compile('\{(.*?)\}|:\((.*)\)')
217 203
218 204 def _extract_route_information(route):
219 205 """
220 206 Convert a route into tuple(name, path, args), eg:
221 207 ('show_user', '/profile/%(username)s', ['username'])
222 208 """
223 209
224 210 routepath = route.pattern
225 211 pattern = route.pattern
226 212
227 213 def replace(matchobj):
228 214 if matchobj.group(1):
229 215 return "%%(%s)s" % matchobj.group(1).split(':')[0]
230 216 else:
231 217 return "%%(%s)s" % matchobj.group(2)
232 218
233 219 routepath = _argument_prog.sub(replace, routepath)
234 220
235 221 if not routepath.startswith('/'):
236 222 routepath = '/'+routepath
237 223
238 224 return (
239 225 route.name,
240 226 routepath,
241 227 [(arg[0].split(':')[0] if arg[0] != '' else arg[1])
242 228 for arg in _argument_prog.findall(pattern)]
243 229 )
244 230
245 231 def get_routes():
246 232 # pyramid routes
247 233 for route in mapper.get_routes():
248 234 if not route.name.startswith('__'):
249 235 yield _extract_route_information(route)
250 236
251 237 if asbool(registry.settings.get('generate_js_files', 'false')):
252 238 static_path = AssetResolver().resolve('rhodecode:public').abspath()
253 239 jsroutes = get_routes()
254 240 jsroutes_file_content = generate_jsroutes_content(jsroutes)
255 241 jsroutes_file_path = os.path.join(
256 242 static_path, 'js', 'rhodecode', 'routes.js')
257 243
258 244 try:
259 245 with io.open(jsroutes_file_path, 'w', encoding='utf-8') as f:
260 246 f.write(jsroutes_file_content)
261 247 except Exception:
262 248 log.exception('Failed to write routes.js into %s', jsroutes_file_path)
263 249
264 250
265 251 class Subscriber(object):
266 252 """
267 253 Base class for subscribers to the pyramid event system.
268 254 """
269 255 def __call__(self, event):
270 256 self.run(event)
271 257
272 258 def run(self, event):
273 259 raise NotImplementedError('Subclass has to implement this.')
274 260
275 261
276 262 class AsyncSubscriber(Subscriber):
277 263 """
278 264 Subscriber that handles the execution of events in a separate task to not
279 265 block the execution of the code which triggers the event. It puts the
280 266 received events into a queue from which the worker process takes them in
281 267 order.
282 268 """
283 269 def __init__(self):
284 270 self._stop = False
285 271 self._eventq = Queue.Queue()
286 272 self._worker = self.create_worker()
287 273 self._worker.start()
288 274
289 275 def __call__(self, event):
290 276 self._eventq.put(event)
291 277
292 278 def create_worker(self):
293 279 worker = Thread(target=self.do_work)
294 280 worker.daemon = True
295 281 return worker
296 282
297 283 def stop_worker(self):
298 284 self._stop = False
299 285 self._eventq.put(None)
300 286 self._worker.join()
301 287
302 288 def do_work(self):
303 289 while not self._stop:
304 290 event = self._eventq.get()
305 291 if event is not None:
306 292 self.run(event)
307 293
308 294
309 295 class AsyncSubprocessSubscriber(AsyncSubscriber):
310 296 """
311 297 Subscriber that uses the subprocess32 module to execute a command if an
312 298 event is received. Events are handled asynchronously.
313 299 """
314 300
315 301 def __init__(self, cmd, timeout=None):
316 302 super(AsyncSubprocessSubscriber, self).__init__()
317 303 self._cmd = cmd
318 304 self._timeout = timeout
319 305
320 306 def run(self, event):
321 307 cmd = self._cmd
322 308 timeout = self._timeout
323 309 log.debug('Executing command %s.', cmd)
324 310
325 311 try:
326 312 output = subprocess32.check_output(
327 313 cmd, timeout=timeout, stderr=subprocess32.STDOUT)
328 314 log.debug('Command finished %s', cmd)
329 315 if output:
330 316 log.debug('Command output: %s', output)
331 317 except subprocess32.TimeoutExpired as e:
332 318 log.exception('Timeout while executing command.')
333 319 if e.output:
334 320 log.error('Command output: %s', e.output)
335 321 except subprocess32.CalledProcessError as e:
336 322 log.exception('Error while executing command.')
337 323 if e.output:
338 324 log.error('Command output: %s', e.output)
339 325 except:
340 326 log.exception(
341 327 'Exception while executing command %s.', cmd)
@@ -1,69 +1,117 b''
1 1 # -*- coding: utf-8 -*-
2 2
3 3 # Copyright (C) 2010-2018 RhodeCode GmbH
4 4 #
5 5 # This program is free software: you can redistribute it and/or modify
6 6 # it under the terms of the GNU Affero General Public License, version 3
7 7 # (only), as published by the Free Software Foundation.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU Affero General Public License
15 15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 16 #
17 17 # This program is dual-licensed. If you wish to learn more about the
18 18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20 20
21 21
22 22 import logging
23 from pyramid.httpexceptions import HTTPException, HTTPBadRequest
23 24
24 25 from rhodecode.lib.middleware.vcs import (
25 26 detect_vcs_request, VCS_TYPE_KEY, VCS_TYPE_SKIP)
26 27
27 28
28 29 log = logging.getLogger(__name__)
29 30
30 31
31 32 def vcs_detection_tween_factory(handler, registry):
32 33
33 34 def vcs_detection_tween(request):
34 35 """
35 36 Do detection of vcs type, and save results for other layers to re-use
36 37 this information
37 38 """
38 39 vcs_server_enabled = request.registry.settings.get('vcs.server.enable')
39 40 vcs_handler = vcs_server_enabled and detect_vcs_request(
40 41 request.environ, request.registry.settings.get('vcs.backends'))
41 42
42 43 if vcs_handler:
43 44 # save detected VCS type for later re-use
44 45 request.environ[VCS_TYPE_KEY] = vcs_handler.SCM
45 46 request.vcs_call = vcs_handler.SCM
46 47
47 48 log.debug('Processing request with `%s` handler', handler)
48 49 return handler(request)
49 50
50 51 # mark that we didn't detect an VCS, and we can skip detection later on
51 52 request.environ[VCS_TYPE_KEY] = VCS_TYPE_SKIP
52 53
53 54 log.debug('Processing request with `%s` handler', handler)
54 55 return handler(request)
55 56
56 57 return vcs_detection_tween
57 58
58 59
60 def junk_encoding_detector(request):
61 """
62 Detect bad encoded GET params, and fail immediately with BadRequest
63 """
64
65 try:
66 request.GET.get("", None)
67 except UnicodeDecodeError:
68 raise HTTPBadRequest("Invalid bytes in query string.")
69
70
71 def bad_url_data_detector(request):
72 """
73 Detect invalid bytes in a path.
74 """
75 try:
76 request.path_info
77 except UnicodeDecodeError:
78 raise HTTPBadRequest("Invalid bytes in URL.")
79
80
81 def junk_form_data_detector(request):
82 """
83 Detect bad encoded POST params, and fail immediately with BadRequest
84 """
85
86 if request.method == "POST":
87 try:
88 request.POST.get("", None)
89 except ValueError:
90 raise HTTPBadRequest("Invalid bytes in form data.")
91
92
93 def sanity_check_factory(handler, registry):
94 def sanity_check(request):
95 try:
96 junk_encoding_detector(request)
97 bad_url_data_detector(request)
98 junk_form_data_detector(request)
99 except HTTPException as exc:
100 return exc
101
102 return handler(request)
103
104 return sanity_check
105
106
59 107 def includeme(config):
60 108 config.add_subscriber('rhodecode.subscribers.add_renderer_globals',
61 109 'pyramid.events.BeforeRender')
62 110 config.add_subscriber('rhodecode.subscribers.set_user_lang',
63 111 'pyramid.events.NewRequest')
64 112 config.add_subscriber('rhodecode.subscribers.add_localizer',
65 113 'pyramid.events.NewRequest')
66 114 config.add_subscriber('rhodecode.subscribers.add_request_user_context',
67 115 'pyramid.events.ContextFound')
68
116 config.add_tween('rhodecode.tweens.sanity_check_factory')
69 117 config.add_tween('rhodecode.tweens.vcs_detection_tween_factory')
General Comments 0
You need to be logged in to leave comments. Login now