# HG changeset patch # User Marcin Kuzminski # Date 2018-02-08 22:38:09 # Node ID 09501d595ca497f0a06886699f40bf55c7b84d8c # Parent 772793de8401d89b8a351bff3177ae85d3fe1742 # Parent 2597b57444b46b2c413eaca27a84e4e67125bf7a release: merge back stable branch into default diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -30,3 +30,4 @@ 22f6744ad8cc274311825f63f953e4dee2ea5cb9 96eb24bea2f5f9258775245e3f09f6fa0a4dda01 v4.10.6 3121217a812c956d7dd5a5875821bd73e8002a32 v4.11.0 fa98b454715ac5b912f39e84af54345909a2a805 v4.11.1 +3982abcfdcc229a723cebe52d3a9bcff10bba08e v4.11.2 diff --git a/docs/release-notes/release-notes-4.11.2.rst b/docs/release-notes/release-notes-4.11.2.rst new file mode 100644 --- /dev/null +++ b/docs/release-notes/release-notes-4.11.2.rst @@ -0,0 +1,47 @@ +|RCE| 4.11.2 |RNS| +------------------ + +Release Date +^^^^^^^^^^^^ + +- 2018-02-08 + + +New Features +^^^^^^^^^^^^ + + + +General +^^^^^^^ + +- git: stabilize long running git processes to prevent potential worker freezes. + + +Security +^^^^^^^^ + +- Repo Checks(low): prevent from html injection on repo-check page. It could + be prepared to fake the displayed text fooling users into unwanted actions. + + +Performance +^^^^^^^^^^^ + + + +Fixes +^^^^^ + +- Diffs: in case of text lexers don't do any HL because of pygments problems. + Fixed newline rendering for certain very unique diffs using Pygments. +- Webhook: fixed extra variable replacement in webhook url. +- Subprocessio: use safe b.read() and prevent potential valueErrors +- Subprocess: use subprocessio helper to run various subprocess commands. +- Git: use subprocessio for hooks execution to use non-blocking subprocess. + + +Upgrade notes +^^^^^^^^^^^^^ + +- Scheduled bugfix release fixing certain requested problems with diffs and, Git. diff --git a/docs/release-notes/release-notes.rst b/docs/release-notes/release-notes.rst --- a/docs/release-notes/release-notes.rst +++ b/docs/release-notes/release-notes.rst @@ -9,6 +9,7 @@ Release Notes .. toctree:: :maxdepth: 1 + release-notes-4.11.2.rst release-notes-4.11.1.rst release-notes-4.11.0.rst release-notes-4.10.6.rst diff --git a/rhodecode/apps/repository/views/repo_checks.py b/rhodecode/apps/repository/views/repo_checks.py --- a/rhodecode/apps/repository/views/repo_checks.py +++ b/rhodecode/apps/repository/views/repo_checks.py @@ -27,6 +27,7 @@ from rhodecode.apps._base import BaseApp from rhodecode.lib import helpers as h from rhodecode.lib.auth import (NotAnonymous, HasRepoPermissionAny) from rhodecode.model.db import Repository +from rhodecode.model.validation_schema.types import RepoNameType log = logging.getLogger(__name__) @@ -43,8 +44,8 @@ class RepoChecksView(BaseAppView): renderer='rhodecode:templates/admin/repos/repo_creating.mako') def repo_creating(self): c = self.load_default_context() - repo_name = self.request.matchdict['repo_name'] + repo_name = RepoNameType().deserialize(None, repo_name) db_repo = Repository.get_by_repo_name(repo_name) # check if maybe repo is already created diff --git a/rhodecode/integrations/types/webhook.py b/rhodecode/integrations/types/webhook.py --- a/rhodecode/integrations/types/webhook.py +++ b/rhodecode/integrations/types/webhook.py @@ -93,13 +93,14 @@ class WebhookHandler(object): 'username': data['actor']['username'], 'user_id': data['actor']['user_id'] } + extra_vars = {} for extra_key, extra_val in data['repo']['extra_fields'].items(): - extra_vars['extra:{}'.format(extra_key)] = extra_val + extra_vars['extra__{}'.format(extra_key)] = extra_val common_vars.update(extra_vars) - return string.Template( - self.template_url).safe_substitute(**common_vars) + template_url = self.template_url.replace('${extra:', '${extra__') + return string.Template(template_url).safe_substitute(**common_vars) def repo_push_event_handler(self, event, data): url = self.get_base_parsed_template(data) diff --git a/rhodecode/lib/codeblocks.py b/rhodecode/lib/codeblocks.py --- a/rhodecode/lib/codeblocks.py +++ b/rhodecode/lib/codeblocks.py @@ -24,6 +24,8 @@ from itertools import groupby from pygments import lex from pygments.formatters.html import _get_ttype_class as pygment_token_class +from pygments.lexers.special import TextLexer, Token + from rhodecode.lib.helpers import ( get_lexer_for_filenode, html_escape, get_custom_lexer) from rhodecode.lib.utils2 import AttributeDict @@ -45,7 +47,7 @@ def filenode_as_lines_tokens(filenode, l log.debug('Generating file node pygment tokens for %s, %s, org_lexer:%s', lexer, filenode, org_lexer) tokens = tokenize_string(filenode.content, lexer) - lines = split_token_stream(tokens, split_string='\n') + lines = split_token_stream(tokens) rv = list(lines) return rv @@ -59,22 +61,28 @@ def tokenize_string(content, lexer): lexer.stripall = False lexer.stripnl = False lexer.ensurenl = False - for token_type, token_text in lex(content, lexer): + + if isinstance(lexer, TextLexer): + lexed = [(Token.Text, content)] + else: + lexed = lex(content, lexer) + + for token_type, token_text in lexed: yield pygment_token_class(token_type), token_text -def split_token_stream(tokens, split_string=u'\n'): +def split_token_stream(tokens): """ Take a list of (TokenType, text) tuples and split them by a string - >>> split_token_stream([(TEXT, 'some\ntext'), (TEXT, 'more\n')]) + split_token_stream([(TEXT, 'some\ntext'), (TEXT, 'more\n')]) [(TEXT, 'some'), (TEXT, 'text'), (TEXT, 'more'), (TEXT, 'text')] """ buffer = [] for token_class, token_text in tokens: - parts = token_text.split(split_string) + parts = token_text.split('\n') for part in parts[:-1]: buffer.append((token_class, part)) yield buffer diff --git a/rhodecode/lib/diffs.py b/rhodecode/lib/diffs.py --- a/rhodecode/lib/diffs.py +++ b/rhodecode/lib/diffs.py @@ -183,13 +183,12 @@ class DiffProcessor(object): :param string: """ - self.cur_diff_size += len(string) if not self.show_full_diff and (self.cur_diff_size > self.diff_limit): raise DiffLimitExceeded('Diff Limit Exceeded') - return safe_unicode(string)\ + return string \ .replace('&', '&')\ .replace('<', '<')\ .replace('>', '>') @@ -278,7 +277,7 @@ class DiffProcessor(object): for chunk in self._diff.chunks(): head = chunk.header - diff = imap(self._escaper, chunk.diff.splitlines(1)) + diff = imap(self._escaper, self.diff_splitter(chunk.diff)) raw_diff = chunk.raw limited_diff = False exceeds_limit = False @@ -529,7 +528,8 @@ class DiffProcessor(object): # a real non-binary diff if head['a_file'] or head['b_file']: - diff = iter(chunk.diff.splitlines(1)) + # simulate splitlines, so we keep the line end part + diff = self.diff_splitter(chunk.diff) # append each file to the diff size raw_chunk_size = len(raw_diff) @@ -608,18 +608,17 @@ class DiffProcessor(object): return diff_container(sorted(_files, key=sorter)) # FIXME: NEWDIFFS: dan: this gets replaced by _new_parse_lines - def _parse_lines(self, diff): + def _parse_lines(self, diff_iter): """ Parse the diff an return data for the template. """ - lineiter = iter(diff) stats = [0, 0] chunks = [] raw_diff = [] try: - line = lineiter.next() + line = diff_iter.next() while line: raw_diff.append(line) @@ -651,7 +650,7 @@ class DiffProcessor(object): 'line': line, }) - line = lineiter.next() + line = diff_iter.next() while old_line < old_end or new_line < new_end: command = ' ' @@ -686,7 +685,7 @@ class DiffProcessor(object): }) raw_diff.append(line) - line = lineiter.next() + line = diff_iter.next() if self._newline_marker.match(line): # we need to append to lines, since this is not @@ -712,13 +711,12 @@ class DiffProcessor(object): chunks = [] raw_diff = [] - diff_iter = imap(lambda s: safe_unicode(s), diff_iter) - try: line = diff_iter.next() while line: raw_diff.append(line) + # match header e.g @@ -0,0 +1 @@\n' match = self._chunk_re.match(line) if not match: @@ -826,6 +824,32 @@ class DiffProcessor(object): idstring = re.sub(r'(?!-)\W', "", idstring).lower() return idstring + @classmethod + def diff_splitter(cls, string): + """ + Diff split that emulates .splitlines() but works only on \n + """ + if not string: + return + elif string == '\n': + yield u'\n' + else: + + has_newline = string.endswith('\n') + elements = string.split('\n') + if has_newline: + # skip last element as it's empty string from newlines + elements = elements[:-1] + + len_elements = len(elements) + + for cnt, line in enumerate(elements, start=1): + last_line = cnt == len_elements + if last_line and not has_newline: + yield safe_unicode(line) + else: + yield safe_unicode(line) + '\n' + def prepare(self, inline_diff=True): """ Prepare the passed udiff for HTML rendering. diff --git a/rhodecode/tests/lib/test_codeblocks.py b/rhodecode/tests/lib/test_codeblocks.py --- a/rhodecode/tests/lib/test_codeblocks.py +++ b/rhodecode/tests/lib/test_codeblocks.py @@ -89,29 +89,9 @@ class TestSplitTokenStream(object): [('type2', u'')], ] - def test_split_token_stream_other_char(self): - lines = list(split_token_stream( - [('type1', 'some\ntext'), ('type2', 'more\n')], - split_string='m')) - - assert lines == [ - [('type1', 'so')], - [('type1', 'e\ntext'), ('type2', '')], - [('type2', 'ore\n')], - ] - - def test_split_token_stream_without_char(self): - lines = list(split_token_stream( - [('type1', 'some\ntext'), ('type2', 'more\n')], - split_string='z')) - - assert lines == [ - [('type1', 'some\ntext'), ('type2', 'more\n')] - ] - def test_split_token_stream_single(self): lines = list(split_token_stream( - [('type1', '\n')], split_string='\n')) + [('type1', '\n')])) assert lines == [ [('type1', '')], @@ -120,7 +100,7 @@ class TestSplitTokenStream(object): def test_split_token_stream_single_repeat(self): lines = list(split_token_stream( - [('type1', '\n\n\n')], split_string='\n')) + [('type1', '\n\n\n')])) assert lines == [ [('type1', '')], @@ -131,7 +111,7 @@ class TestSplitTokenStream(object): def test_split_token_stream_multiple_repeat(self): lines = list(split_token_stream( - [('type1', '\n\n'), ('type2', '\n\n')], split_string='\n')) + [('type1', '\n\n'), ('type2', '\n\n')])) assert lines == [ [('type1', '')], diff --git a/rhodecode/tests/lib/test_diffs.py b/rhodecode/tests/lib/test_diffs.py --- a/rhodecode/tests/lib/test_diffs.py +++ b/rhodecode/tests/lib/test_diffs.py @@ -26,7 +26,7 @@ from rhodecode.lib.diffs import ( DiffProcessor, NEW_FILENODE, DEL_FILENODE, MOD_FILENODE, RENAMED_FILENODE, CHMOD_FILENODE, BIN_FILENODE, COPIED_FILENODE) -from rhodecode.tests.fixture import Fixture +from rhodecode.tests.fixture import Fixture, no_newline_id_generator from rhodecode.lib.vcs.backends.git.repository import GitDiff from rhodecode.lib.vcs.backends.hg.repository import MercurialDiff from rhodecode.lib.vcs.backends.svn.repository import SubversionDiff @@ -162,7 +162,7 @@ def test_diffprocessor_as_html_with_comm assert html == expected_html -class TestMixedFilenameEncodings: +class TestMixedFilenameEncodings(object): @pytest.fixture(scope="class") def raw_diff(self): @@ -811,3 +811,21 @@ def test_diff_lib_newlines(diff_fixture_ data = [(x['filename'], x['operation'], x['stats'], x['raw_diff']) for x in diff_proc_d] assert expected_data == data + + +@pytest.mark.parametrize('input_str', [ + '', + '\n', + '\n\n', + 'First\n+second', + 'First\n+second\n', + + '\n\n\n Multi \n\n\n', + '\n\n\n Multi beginning', + 'Multi end \n\n\n', + 'Multi end', + '@@ -0,0 +1 @@\n+test_content \n\n b\n' +], ids=no_newline_id_generator) +def test_splitlines(input_str): + result = DiffProcessor.diff_splitter(input_str) + assert list(result) == input_str.splitlines(True)