release: merge back stable branch into default

docs/release-notes/release-notes-4.15.1.rst

@@ -0,0 +1,50 b''
+|RCE| 4.15.1 |RNS|
+------------------
+
+Release Date
+^^^^^^^^^^^^
+
+- 2019-01-01
+
+
+New Features
+^^^^^^^^^^^^
+
+
+
+General
+^^^^^^^
+
+- Downloads: properly encode " in the filenames, and add RFC 5987 header for non-ascii files.
+- Documentation: updated configuration for Nginx and reverse proxy.
+- VCS: streaming will use now 100kb chunks for faster network throughput.
+
+
+Security
+^^^^^^^^
+
+- Diffs: fixed xss in context diff menu.
+- Downloads: properly encode " in the filenames, prevents from hiding executable
+  files disguised in another type of file using crafted file names.
+
+Performance
+^^^^^^^^^^^
+
+
+
+Fixes
+^^^^^
+
+- VCS: handle excessive slashes in from of the repo name path, fixes #5522.
+  This prevents 500 errors when excessive slashes are used
+- SVN: support proxy-prefix properly, fixes #5521.
+- Pull requests: validate ref types on API calls for pull request so users cannot
+  provide wrongs ones.
+- Scheduler: fix url generation with proxy prefix.
+- Celery: add DB connection ping to validate DB connection is working at worker startup.
+
+
+Upgrade notes
+^^^^^^^^^^^^^
+
+- Scheduled release addressing reported problems in 4.15.X releases.

.hgtags

@@ -47,3 +47,4 b' 7dc62c090881fb5d03268141e71e0940d7c3295d'
 9151328c1c46b72ba6f00d7640d9141e75aa1ca2 v4.14.0
 a47eeac5dfa41fa6779d90452affba4091c3ade8 v4.14.1
 4b34ce0d2c3c10510626b3b65044939bb7a2cddf v4.15.0
[phone]d22e6b70613674cd675ae9a604b7989f v4.15.1

docs/release-notes/release-notes.rst

@@ -9,6 +9,7 b' Release Notes'
 .. toctree::
    :maxdepth: 1
 
+   release-notes-4.15.1.rst
    release-notes-4.15.0.rst
    release-notes-4.14.1.rst
    release-notes-4.14.0.rst

rhodecode/api/tests/test_create_pull_request.py

@@ -56,6 +56,25 b' class TestCreatePullRequestApi(object):'
             assert_error(id_, expected, given=response.body)
 
     @pytest.mark.backends("git", "hg")
+    @pytest.mark.parametrize('source_ref', [
+        'bookmarg:default:initial'
+    ])
+    def test_create_with_wrong_refs_data(self, backend, source_ref):
+
+        data = self._prepare_data(backend)
+        data['source_ref'] = source_ref
+
+        id_, params = build_data(
+            self.apikey_regular, 'create_pull_request', **data)
+
+        response = api_call(self.app, params)
+
+        expected = "Ref `{}` type is not allowed. " \
+                   "Only:['bookmark', 'book', 'tag', 'branch'] " \
+                   "are possible.".format(source_ref)
+        assert_error(id_, expected, given=response.body)
+
+    @pytest.mark.backends("git", "hg")
     def test_create_with_correct_data(self, backend):
         data = self._prepare_data(backend)
         RepoModel().revoke_user_permission(

rhodecode/api/tests/test_utils.py

@@ -84,11 +84,11 b' class TestResolveRefOrError(object):'
 
     def test_non_supported_refs(self):
         repo = Mock()
-        ref = 'ancestor:ref'
+        ref = 'bookmark:ref'
         with pytest.raises(JSONRPCError) as excinfo:
             utils.resolve_ref_or_error(ref, repo)
         expected_message = (
-            'The specified value:ancestor:`ref` does not exist, or is not allowed.')
+            'The specified value:bookmark:`ref` does not exist, or is not allowed.')
         assert excinfo.value.message == expected_message
 
     def test_branch_is_not_found(self):

rhodecode/api/utils.py

@@ -388,7 +388,19 b' def get_commit_or_error(ref, repo):'
         raise JSONRPCError('Ref `{ref}` does not exist'.format(ref=ref))
 
 
-def resolve_ref_or_error(ref, repo):
+def _get_ref_hash(repo, type_, name):
+    vcs_repo = repo.scm_instance()
+    if type_ in ['branch'] and vcs_repo.alias in ('hg', 'git'):
+        return vcs_repo.branches[name]
+    elif type_ in ['bookmark', 'book'] and vcs_repo.alias == 'hg':
+        return vcs_repo.bookmarks[name]
+    else:
+        raise ValueError()
+
+
+def resolve_ref_or_error(ref, repo, allowed_ref_types=None):
+    allowed_ref_types = allowed_ref_types or ['bookmark', 'book', 'tag', 'branch']
+
     def _parse_ref(type_, name, hash_=None):
         return type_, name, hash_
 
@@ -399,6 +411,12 b' def resolve_ref_or_error(ref, repo):'
             'Ref `{ref}` given in a wrong format. Please check the API'
             ' documentation for more details'.format(ref=ref))
 
+    if ref_type not in allowed_ref_types:
+        raise JSONRPCError(
+            'Ref `{ref}` type is not allowed. '
+            'Only:{allowed_refs} are possible.'.format(
+                ref=ref, allowed_refs=allowed_ref_types))
+
     try:
         ref_hash = ref_hash or _get_ref_hash(repo, ref_type, ref_name)
     except (KeyError, ValueError):
@@ -429,13 +447,3 b' def _get_commit_dict('
         "raw_diff": raw_diff,
         "stats": stats
     }
-
-
-def _get_ref_hash(repo, type_, name):
-    vcs_repo = repo.scm_instance()
-    if type_ == 'branch' and vcs_repo.alias in ('hg', 'git'):
-        return vcs_repo.branches[name]
-    elif type_ == 'bookmark' and vcs_repo.alias == 'hg':
-        return vcs_repo.bookmarks[name]
-    else:
-        raise ValueError()

rhodecode/apps/repository/tests/test_repo_files.py

@@ -428,7 +428,7 b' class TestRawFileHandling(object):'
                        repo_name=backend.repo_name,
                        commit_id=commit.raw_id, f_path='vcs/nodes.py'),)
 
-        assert response.content_disposition == "attachment; filename=nodes.py"
+        assert response.content_disposition == 'attachment; filename="nodes.py"; filename*=UTF-8\'\'nodes.py'
         assert response.content_type == "text/x-python"
 
     def test_download_file_wrong_cs(self, backend):

rhodecode/apps/repository/views/repo_files.py

@@ -24,6 +24,7 b' import os'
 import shutil
 import tempfile
 import collections
+import urllib
 
 from pyramid.httpexceptions import HTTPNotFound, HTTPBadRequest, HTTPFound
 from pyramid.view import view_config
@@ -709,9 +710,14 b' class RepoFilesView(RepoAppView):'
 
         return Response(html)
 
-    def _get_attachement_disposition(self, f_path):
-        return 'attachment; filename=%s' % \
-            safe_str(f_path.split(Repository.NAME_SEP)[-1])
+    def _get_attachement_headers(self, f_path):
+        f_name = safe_str(f_path.split(Repository.NAME_SEP)[-1])
+        safe_path = f_name.replace('"', '\\"')
+        encoded_path = urllib.quote(f_name)
+
+        return "attachment; " \
+               "filename=\"{}\"; " \
+               "filename*=UTF-8\'\'{}".format(safe_path, encoded_path)
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
@@ -766,7 +772,7 b' class RepoFilesView(RepoAppView):'
                 mimetype, disposition = 'text/plain', 'inline'
 
         if disposition == 'attachment':
-            disposition = self._get_attachement_disposition(f_path)
+            disposition = self._get_attachement_headers(f_path)
 
         def stream_node():
             yield file_node.raw_bytes
@@ -805,7 +811,7 b' class RepoFilesView(RepoAppView):'
                 # overwrite our pointer with the REAL large-file
                 file_node = lf_node
 
-        disposition = self._get_attachement_disposition(f_path)
+        disposition = self._get_attachement_headers(f_path)
 
         def stream_node():
             yield file_node.raw_bytes

rhodecode/lib/middleware/simplesvn.py

@@ -51,7 +51,8 b' class SimpleSvnApp(object):'
         data = environ['wsgi.input']
         req_method = environ['REQUEST_METHOD']
         has_content_length = 'CONTENT_LENGTH' in environ
-        path_info = self._get_url(environ['PATH_INFO'])
+        path_info = self._get_url(
+            self.config.get('subversion_http_server_url', ''), environ['PATH_INFO'])
         transfer_encoding = environ.get('HTTP_TRANSFER_ENCODING', '')
         log.debug('Handling: %s method via `%s`', req_method, path_info)
 
@@ -117,9 +118,9 b' class SimpleSvnApp(object):'
             response_headers)
         return response.iter_content(chunk_size=1024)
 
-    def _get_url(self, path):
-        url_path = urlparse.urljoin(
-            self.config.get('subversion_http_server_url', ''), path)
+    def _get_url(self, svn_http_server, path):
+        svn_http_server_url = (svn_http_server or '').rstrip('/')
+        url_path = urlparse.urljoin(svn_http_server_url + '/', (path or '').lstrip('/'))
         url_path = urllib.quote(url_path, safe="/:=~+!$,;'")
         return url_path
 

rhodecode/lib/utils2.py

@@ -43,6 +43,7 b' import sqlalchemy.exc'
 import sqlalchemy.sql
 import webob
 import pyramid.threadlocal
+from pyramid.settings import asbool
 
 import rhodecode
 from rhodecode.translation import _, _pluralize
@@ -361,7 +362,8 b' def ping_connection(connection, branch):'
 def engine_from_config(configuration, prefix='sqlalchemy.', **kwargs):
     """Custom engine_from_config functions."""
     log = logging.getLogger('sqlalchemy.engine')
-    _ping_connection = configuration.pop('sqlalchemy.db1.ping_connection', None)
+    use_ping_connection = asbool(configuration.pop('sqlalchemy.db1.ping_connection', None))
+    debug = asbool(configuration.get('debug'))
 
     engine = sqlalchemy.engine_from_config(configuration, prefix, **kwargs)
 
@@ -370,12 +372,12 b' def engine_from_config(configuration, pr'
         normal = '\x1b[0m'
         return ''.join([color_seq, sql, normal])
 
-    if configuration['debug'] or _ping_connection:
+    if use_ping_connection:
+        log.debug('Adding ping_connection on the engine config.')
         sqlalchemy.event.listen(engine, "engine_connect", ping_connection)
 
-    if configuration['debug']:
+    if debug:
         # attach events only for debug configuration
-
         def before_cursor_execute(conn, cursor, statement,
                                   parameters, context, executemany):
             setattr(conn, 'query_start_time', time.time())
@@ -394,10 +396,8 b' def engine_from_config(configuration, pr'
                                  parameters, context, executemany):
             delattr(conn, 'query_start_time')
 
-        sqlalchemy.event.listen(engine, "before_cursor_execute",
-                                before_cursor_execute)
-        sqlalchemy.event.listen(engine, "after_cursor_execute",
-                                after_cursor_execute)
+        sqlalchemy.event.listen(engine, "before_cursor_execute", before_cursor_execute)
+        sqlalchemy.event.listen(engine, "after_cursor_execute", after_cursor_execute)
 
     return engine
 

rhodecode/model/pull_request.py

@@ -92,6 +92,8 b' class PullRequestModel(BaseModel):'
             'This pull request cannot be updated because the source '
             'reference is missing.'),
     }
+    REF_TYPES = ['bookmark', 'book', 'tag', 'branch']
+    UPDATABLE_REF_TYPES = ['bookmark', 'book', 'branch']
 
     def __get_pull_request(self, pull_request):
         return self._get_instance((
@@ -633,7 +635,7 b' class PullRequestModel(BaseModel):'
 
     def has_valid_update_type(self, pull_request):
         source_ref_type = pull_request.source_ref_parts.type
-        return source_ref_type in ['book', 'branch', 'tag']
+        return source_ref_type in self.REF_TYPES
 
     def update_commits(self, pull_request):
         """
@@ -713,7 +715,7 b' class PullRequestModel(BaseModel):'
             pull_request_version = pull_request
 
         try:
-            if target_ref_type in ('tag', 'branch', 'book'):
+            if target_ref_type in self.REF_TYPES:
                 target_commit = target_repo.get_commit(target_ref_name)
             else:
                 target_commit = target_repo.get_commit(target_ref_id)
@@ -1289,7 +1291,7 b' class PullRequestModel(BaseModel):'
         return merge_state
 
     def _refresh_reference(self, reference, vcs_repository):
-        if reference.type in ('branch', 'book'):
+        if reference.type in self.UPDATABLE_REF_TYPES:
             name_or_id = reference.name
         else:
             name_or_id = reference.commit_id

rhodecode/templates/codeblocks/diffs.mako

@@ -909,6 +909,8 b' def get_comments_for(diff_type, comments'
             };
 
             var animateText =  $.debounce(100, function(fPath, anchorId) {
+                fPath = Select2.util.escapeMarkup(fPath);
+
                 // animate setting the text
                 var callback = function () {
                     $('.fpath-placeholder-text').animate({'opacity': 1.00}, 200)

rhodecode/tests/lib/middleware/test_simplesvn.py

@@ -161,9 +161,17 b' class TestSimpleSvnApp(object):'
         response_headers = self.app._get_response_headers(headers)
         assert sorted(response_headers) == sorted(expected_headers)
 
-    def test_get_url(self):
-        url = self.app._get_url(self.path)
-        expected_url = '{}{}'.format(self.host.strip('/'), self.path)
+    @pytest.mark.parametrize('svn_http_url, path_info, expected_url', [
+        ('http://localhost:8200', '/repo_name', 'http://localhost:8200/repo_name'),
+        ('http://localhost:8200///', '/repo_name', 'http://localhost:8200/repo_name'),
+        ('http://localhost:8200', '/group/repo_name', 'http://localhost:8200/group/repo_name'),
+        ('http://localhost:8200/', '/group/repo_name', 'http://localhost:8200/group/repo_name'),
+        ('http://localhost:8200/prefix', '/repo_name', 'http://localhost:8200/prefix/repo_name'),
+        ('http://localhost:8200/prefix', 'repo_name', 'http://localhost:8200/prefix/repo_name'),
+        ('http://localhost:8200/prefix', '/group/repo_name', 'http://localhost:8200/prefix/group/repo_name')
+    ])
+    def test_get_url(self, svn_http_url, path_info, expected_url):
+        url = self.app._get_url(svn_http_url, path_info)
         assert url == expected_url
 
     def test_call(self):