# HG changeset patch # User Marcin Kuzminski # Date 2019-01-02 16:04:25 # Node ID ac6efde5d1351234a1e07f20e3764de30b686d60 # Parent 9cc1ff03115e2e0707c6c12c96fda6dc03c88f8f # Parent 8fb8e14ef0b20596589389f06f99e83c1b1bb2e6 release: merge back stable branch into default diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -47,3 +47,4 @@ 7dc62c090881fb5d03268141e71e0940d7c3295d 9151328c1c46b72ba6f00d7640d9141e75aa1ca2 v4.14.0 a47eeac5dfa41fa6779d90452affba4091c3ade8 v4.14.1 4b34ce0d2c3c10510626b3b65044939bb7a2cddf v4.15.0 +14502561d22e6b70613674cd675ae9a604b7989f v4.15.1 diff --git a/docs/release-notes/release-notes-4.15.1.rst b/docs/release-notes/release-notes-4.15.1.rst new file mode 100644 --- /dev/null +++ b/docs/release-notes/release-notes-4.15.1.rst @@ -0,0 +1,50 @@ +|RCE| 4.15.1 |RNS| +------------------ + +Release Date +^^^^^^^^^^^^ + +- 2019-01-01 + + +New Features +^^^^^^^^^^^^ + + + +General +^^^^^^^ + +- Downloads: properly encode " in the filenames, and add RFC 5987 header for non-ascii files. +- Documentation: updated configuration for Nginx and reverse proxy. +- VCS: streaming will use now 100kb chunks for faster network throughput. + + +Security +^^^^^^^^ + +- Diffs: fixed xss in context diff menu. +- Downloads: properly encode " in the filenames, prevents from hiding executable + files disguised in another type of file using crafted file names. + +Performance +^^^^^^^^^^^ + + + +Fixes +^^^^^ + +- VCS: handle excessive slashes in from of the repo name path, fixes #5522. + This prevents 500 errors when excessive slashes are used +- SVN: support proxy-prefix properly, fixes #5521. +- Pull requests: validate ref types on API calls for pull request so users cannot + provide wrongs ones. +- Scheduler: fix url generation with proxy prefix. +- Celery: add DB connection ping to validate DB connection is working at worker startup. + + +Upgrade notes +^^^^^^^^^^^^^ + +- Scheduled release addressing reported problems in 4.15.X releases. diff --git a/docs/release-notes/release-notes.rst b/docs/release-notes/release-notes.rst --- a/docs/release-notes/release-notes.rst +++ b/docs/release-notes/release-notes.rst @@ -9,6 +9,7 @@ Release Notes .. toctree:: :maxdepth: 1 + release-notes-4.15.1.rst release-notes-4.15.0.rst release-notes-4.14.1.rst release-notes-4.14.0.rst diff --git a/rhodecode/api/tests/test_create_pull_request.py b/rhodecode/api/tests/test_create_pull_request.py --- a/rhodecode/api/tests/test_create_pull_request.py +++ b/rhodecode/api/tests/test_create_pull_request.py @@ -56,6 +56,25 @@ class TestCreatePullRequestApi(object): assert_error(id_, expected, given=response.body) @pytest.mark.backends("git", "hg") + @pytest.mark.parametrize('source_ref', [ + 'bookmarg:default:initial' + ]) + def test_create_with_wrong_refs_data(self, backend, source_ref): + + data = self._prepare_data(backend) + data['source_ref'] = source_ref + + id_, params = build_data( + self.apikey_regular, 'create_pull_request', **data) + + response = api_call(self.app, params) + + expected = "Ref `{}` type is not allowed. " \ + "Only:['bookmark', 'book', 'tag', 'branch'] " \ + "are possible.".format(source_ref) + assert_error(id_, expected, given=response.body) + + @pytest.mark.backends("git", "hg") def test_create_with_correct_data(self, backend): data = self._prepare_data(backend) RepoModel().revoke_user_permission( diff --git a/rhodecode/api/tests/test_utils.py b/rhodecode/api/tests/test_utils.py --- a/rhodecode/api/tests/test_utils.py +++ b/rhodecode/api/tests/test_utils.py @@ -84,11 +84,11 @@ class TestResolveRefOrError(object): def test_non_supported_refs(self): repo = Mock() - ref = 'ancestor:ref' + ref = 'bookmark:ref' with pytest.raises(JSONRPCError) as excinfo: utils.resolve_ref_or_error(ref, repo) expected_message = ( - 'The specified value:ancestor:`ref` does not exist, or is not allowed.') + 'The specified value:bookmark:`ref` does not exist, or is not allowed.') assert excinfo.value.message == expected_message def test_branch_is_not_found(self): diff --git a/rhodecode/api/utils.py b/rhodecode/api/utils.py --- a/rhodecode/api/utils.py +++ b/rhodecode/api/utils.py @@ -388,7 +388,19 @@ def get_commit_or_error(ref, repo): raise JSONRPCError('Ref `{ref}` does not exist'.format(ref=ref)) -def resolve_ref_or_error(ref, repo): +def _get_ref_hash(repo, type_, name): + vcs_repo = repo.scm_instance() + if type_ in ['branch'] and vcs_repo.alias in ('hg', 'git'): + return vcs_repo.branches[name] + elif type_ in ['bookmark', 'book'] and vcs_repo.alias == 'hg': + return vcs_repo.bookmarks[name] + else: + raise ValueError() + + +def resolve_ref_or_error(ref, repo, allowed_ref_types=None): + allowed_ref_types = allowed_ref_types or ['bookmark', 'book', 'tag', 'branch'] + def _parse_ref(type_, name, hash_=None): return type_, name, hash_ @@ -399,6 +411,12 @@ def resolve_ref_or_error(ref, repo): 'Ref `{ref}` given in a wrong format. Please check the API' ' documentation for more details'.format(ref=ref)) + if ref_type not in allowed_ref_types: + raise JSONRPCError( + 'Ref `{ref}` type is not allowed. ' + 'Only:{allowed_refs} are possible.'.format( + ref=ref, allowed_refs=allowed_ref_types)) + try: ref_hash = ref_hash or _get_ref_hash(repo, ref_type, ref_name) except (KeyError, ValueError): @@ -429,13 +447,3 @@ def _get_commit_dict( "raw_diff": raw_diff, "stats": stats } - - -def _get_ref_hash(repo, type_, name): - vcs_repo = repo.scm_instance() - if type_ == 'branch' and vcs_repo.alias in ('hg', 'git'): - return vcs_repo.branches[name] - elif type_ == 'bookmark' and vcs_repo.alias == 'hg': - return vcs_repo.bookmarks[name] - else: - raise ValueError() diff --git a/rhodecode/apps/repository/tests/test_repo_files.py b/rhodecode/apps/repository/tests/test_repo_files.py --- a/rhodecode/apps/repository/tests/test_repo_files.py +++ b/rhodecode/apps/repository/tests/test_repo_files.py @@ -428,7 +428,7 @@ class TestRawFileHandling(object): repo_name=backend.repo_name, commit_id=commit.raw_id, f_path='vcs/nodes.py'),) - assert response.content_disposition == "attachment; filename=nodes.py" + assert response.content_disposition == 'attachment; filename="nodes.py"; filename*=UTF-8\'\'nodes.py' assert response.content_type == "text/x-python" def test_download_file_wrong_cs(self, backend): diff --git a/rhodecode/apps/repository/views/repo_files.py b/rhodecode/apps/repository/views/repo_files.py --- a/rhodecode/apps/repository/views/repo_files.py +++ b/rhodecode/apps/repository/views/repo_files.py @@ -24,6 +24,7 @@ import os import shutil import tempfile import collections +import urllib from pyramid.httpexceptions import HTTPNotFound, HTTPBadRequest, HTTPFound from pyramid.view import view_config @@ -709,9 +710,14 @@ class RepoFilesView(RepoAppView): return Response(html) - def _get_attachement_disposition(self, f_path): - return 'attachment; filename=%s' % \ - safe_str(f_path.split(Repository.NAME_SEP)[-1]) + def _get_attachement_headers(self, f_path): + f_name = safe_str(f_path.split(Repository.NAME_SEP)[-1]) + safe_path = f_name.replace('"', '\\"') + encoded_path = urllib.quote(f_name) + + return "attachment; " \ + "filename=\"{}\"; " \ + "filename*=UTF-8\'\'{}".format(safe_path, encoded_path) @LoginRequired() @HasRepoPermissionAnyDecorator( @@ -766,7 +772,7 @@ class RepoFilesView(RepoAppView): mimetype, disposition = 'text/plain', 'inline' if disposition == 'attachment': - disposition = self._get_attachement_disposition(f_path) + disposition = self._get_attachement_headers(f_path) def stream_node(): yield file_node.raw_bytes @@ -805,7 +811,7 @@ class RepoFilesView(RepoAppView): # overwrite our pointer with the REAL large-file file_node = lf_node - disposition = self._get_attachement_disposition(f_path) + disposition = self._get_attachement_headers(f_path) def stream_node(): yield file_node.raw_bytes diff --git a/rhodecode/lib/middleware/simplesvn.py b/rhodecode/lib/middleware/simplesvn.py --- a/rhodecode/lib/middleware/simplesvn.py +++ b/rhodecode/lib/middleware/simplesvn.py @@ -51,7 +51,8 @@ class SimpleSvnApp(object): data = environ['wsgi.input'] req_method = environ['REQUEST_METHOD'] has_content_length = 'CONTENT_LENGTH' in environ - path_info = self._get_url(environ['PATH_INFO']) + path_info = self._get_url( + self.config.get('subversion_http_server_url', ''), environ['PATH_INFO']) transfer_encoding = environ.get('HTTP_TRANSFER_ENCODING', '') log.debug('Handling: %s method via `%s`', req_method, path_info) @@ -117,9 +118,9 @@ class SimpleSvnApp(object): response_headers) return response.iter_content(chunk_size=1024) - def _get_url(self, path): - url_path = urlparse.urljoin( - self.config.get('subversion_http_server_url', ''), path) + def _get_url(self, svn_http_server, path): + svn_http_server_url = (svn_http_server or '').rstrip('/') + url_path = urlparse.urljoin(svn_http_server_url + '/', (path or '').lstrip('/')) url_path = urllib.quote(url_path, safe="/:=~+!$,;'") return url_path diff --git a/rhodecode/lib/utils2.py b/rhodecode/lib/utils2.py --- a/rhodecode/lib/utils2.py +++ b/rhodecode/lib/utils2.py @@ -43,6 +43,7 @@ import sqlalchemy.exc import sqlalchemy.sql import webob import pyramid.threadlocal +from pyramid.settings import asbool import rhodecode from rhodecode.translation import _, _pluralize @@ -361,7 +362,8 @@ def ping_connection(connection, branch): def engine_from_config(configuration, prefix='sqlalchemy.', **kwargs): """Custom engine_from_config functions.""" log = logging.getLogger('sqlalchemy.engine') - _ping_connection = configuration.pop('sqlalchemy.db1.ping_connection', None) + use_ping_connection = asbool(configuration.pop('sqlalchemy.db1.ping_connection', None)) + debug = asbool(configuration.get('debug')) engine = sqlalchemy.engine_from_config(configuration, prefix, **kwargs) @@ -370,12 +372,12 @@ def engine_from_config(configuration, pr normal = '\x1b[0m' return ''.join([color_seq, sql, normal]) - if configuration['debug'] or _ping_connection: + if use_ping_connection: + log.debug('Adding ping_connection on the engine config.') sqlalchemy.event.listen(engine, "engine_connect", ping_connection) - if configuration['debug']: + if debug: # attach events only for debug configuration - def before_cursor_execute(conn, cursor, statement, parameters, context, executemany): setattr(conn, 'query_start_time', time.time()) @@ -394,10 +396,8 @@ def engine_from_config(configuration, pr parameters, context, executemany): delattr(conn, 'query_start_time') - sqlalchemy.event.listen(engine, "before_cursor_execute", - before_cursor_execute) - sqlalchemy.event.listen(engine, "after_cursor_execute", - after_cursor_execute) + sqlalchemy.event.listen(engine, "before_cursor_execute", before_cursor_execute) + sqlalchemy.event.listen(engine, "after_cursor_execute", after_cursor_execute) return engine diff --git a/rhodecode/model/pull_request.py b/rhodecode/model/pull_request.py --- a/rhodecode/model/pull_request.py +++ b/rhodecode/model/pull_request.py @@ -92,6 +92,8 @@ class PullRequestModel(BaseModel): 'This pull request cannot be updated because the source ' 'reference is missing.'), } + REF_TYPES = ['bookmark', 'book', 'tag', 'branch'] + UPDATABLE_REF_TYPES = ['bookmark', 'book', 'branch'] def __get_pull_request(self, pull_request): return self._get_instance(( @@ -633,7 +635,7 @@ class PullRequestModel(BaseModel): def has_valid_update_type(self, pull_request): source_ref_type = pull_request.source_ref_parts.type - return source_ref_type in ['book', 'branch', 'tag'] + return source_ref_type in self.REF_TYPES def update_commits(self, pull_request): """ @@ -713,7 +715,7 @@ class PullRequestModel(BaseModel): pull_request_version = pull_request try: - if target_ref_type in ('tag', 'branch', 'book'): + if target_ref_type in self.REF_TYPES: target_commit = target_repo.get_commit(target_ref_name) else: target_commit = target_repo.get_commit(target_ref_id) @@ -1289,7 +1291,7 @@ class PullRequestModel(BaseModel): return merge_state def _refresh_reference(self, reference, vcs_repository): - if reference.type in ('branch', 'book'): + if reference.type in self.UPDATABLE_REF_TYPES: name_or_id = reference.name else: name_or_id = reference.commit_id diff --git a/rhodecode/templates/codeblocks/diffs.mako b/rhodecode/templates/codeblocks/diffs.mako --- a/rhodecode/templates/codeblocks/diffs.mako +++ b/rhodecode/templates/codeblocks/diffs.mako @@ -909,6 +909,8 @@ def get_comments_for(diff_type, comments }; var animateText = $.debounce(100, function(fPath, anchorId) { + fPath = Select2.util.escapeMarkup(fPath); + // animate setting the text var callback = function () { $('.fpath-placeholder-text').animate({'opacity': 1.00}, 200) diff --git a/rhodecode/tests/lib/middleware/test_simplesvn.py b/rhodecode/tests/lib/middleware/test_simplesvn.py --- a/rhodecode/tests/lib/middleware/test_simplesvn.py +++ b/rhodecode/tests/lib/middleware/test_simplesvn.py @@ -161,9 +161,17 @@ class TestSimpleSvnApp(object): response_headers = self.app._get_response_headers(headers) assert sorted(response_headers) == sorted(expected_headers) - def test_get_url(self): - url = self.app._get_url(self.path) - expected_url = '{}{}'.format(self.host.strip('/'), self.path) + @pytest.mark.parametrize('svn_http_url, path_info, expected_url', [ + ('http://localhost:8200', '/repo_name', 'http://localhost:8200/repo_name'), + ('http://localhost:8200///', '/repo_name', 'http://localhost:8200/repo_name'), + ('http://localhost:8200', '/group/repo_name', 'http://localhost:8200/group/repo_name'), + ('http://localhost:8200/', '/group/repo_name', 'http://localhost:8200/group/repo_name'), + ('http://localhost:8200/prefix', '/repo_name', 'http://localhost:8200/prefix/repo_name'), + ('http://localhost:8200/prefix', 'repo_name', 'http://localhost:8200/prefix/repo_name'), + ('http://localhost:8200/prefix', '/group/repo_name', 'http://localhost:8200/prefix/group/repo_name') + ]) + def test_get_url(self, svn_http_url, path_info, expected_url): + url = self.app._get_url(svn_http_url, path_info) assert url == expected_url def test_call(self):