core: use application wide detection of invalid bytes sent via URL/GET/POST data.

rhodecode/tests/functional/test_bad_request_data.py

@@ -0,0 +1,45 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2016-2018 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+import pytest
+
+from rhodecode.tests import TestController
+from rhodecode.tests.fixture import Fixture
+
+
+class TestBadRequestData(TestController):
+
+    def test_bad_get_data(self):
+        self.app.get(
+            '/', params={'f\xfc': '\xfc%f6%22%20onmouseover%3dveA2(9352)%20'},
+            status=400)
+
+    def test_bad_url_data(self):
+        self.app.post(
+            '/f\xfc',
+            status=400)
+
+    def test_bad_post_data(self, csrf_token, xhr_header):
+        self.app.post(
+            '/_markup_preview',
+            params={'f\xfc': '\xfc%f6%22%20onmouseover%3dveA2(9352)%20',
+                    'csrf_token': csrf_token},
+            extra_environ=xhr_header,
+            status=400)

rhodecode/subscribers.py

@@ -27,7 +27,6 b' import subprocess32'
 
 
 from dateutil.parser import parse
-from pyramid.httpexceptions import HTTPBadRequest
 from pyramid.threadlocal import get_current_request
 from pyramid.interfaces import IRoutesMapper
 from pyramid.settings import asbool
@@ -39,7 +38,6 b' from rhodecode.config.jsroutes import ge'
 from rhodecode.lib import auth
 from rhodecode.lib.base import get_auth_user
 
-
 import rhodecode
 
 
@@ -61,19 +59,7 b' def add_renderer_globals(event):'
 
 def add_localizer(event):
     request = event.request
-    try:
-        localizer = request.localizer
-    except Exception:
-        log.exception('Failed to get localizer')
-        # NOTE(marcink): Handle bug in pyramid by malformed GET params could crash
-        # this resulting on various odd errors on missing translators
-        # see: https://github.com/Pylons/pyramid/issues/3399
-
-        def dummy_translate(*args, **kwargs):
-            return ''.join(args)
-        request.translate = tsf
-        request.plularize = dummy_translate
-        raise HTTPBadRequest()
+    localizer = request.localizer
 
     def auto_translate(*args, **kwargs):
         return localizer.translate(tsf(*args, **kwargs))

rhodecode/tweens.py

@@ -20,6 +20,7 b''
 
 
 import logging
+from pyramid.httpexceptions import HTTPException, HTTPBadRequest
 
 from rhodecode.lib.middleware.vcs import (
     detect_vcs_request, VCS_TYPE_KEY, VCS_TYPE_SKIP)
@@ -56,6 +57,53 b' def vcs_detection_tween_factory(handler,'
     return vcs_detection_tween
 
 
+def junk_encoding_detector(request):
+    """
+    Detect bad encoded GET params, and fail immediately with BadRequest
+    """
+
+    try:
+        request.GET.get("", None)
+    except UnicodeDecodeError:
+        raise HTTPBadRequest("Invalid bytes in query string.")
+
+
+def bad_url_data_detector(request):
+    """
+    Detect invalid bytes in a path.
+    """
+    try:
+        request.path_info
+    except UnicodeDecodeError:
+        raise HTTPBadRequest("Invalid bytes in URL.")
+
+
+def junk_form_data_detector(request):
+    """
+    Detect bad encoded POST params, and fail immediately with BadRequest
+    """
+
+    if request.method == "POST":
+        try:
+            request.POST.get("", None)
+        except ValueError:
+            raise HTTPBadRequest("Invalid bytes in form data.")
+
+
+def sanity_check_factory(handler, registry):
+    def sanity_check(request):
+        try:
+            junk_encoding_detector(request)
+            bad_url_data_detector(request)
+            junk_form_data_detector(request)
+        except HTTPException as exc:
+            return exc
+
+        return handler(request)
+
+    return sanity_check
+
+
 def includeme(config):
     config.add_subscriber('rhodecode.subscribers.add_renderer_globals',
                           'pyramid.events.BeforeRender')
@@ -65,5 +113,5 b' def includeme(config):'
                           'pyramid.events.NewRequest')
     config.add_subscriber('rhodecode.subscribers.add_request_user_context',
                           'pyramid.events.ContextFound')
-
+    config.add_tween('rhodecode.tweens.sanity_check_factory')
     config.add_tween('rhodecode.tweens.vcs_detection_tween_factory')