##// END OF EJS Templates
Merge pull request #4504 from takluyver/inputtransformer-syntaxerror...
Fernando Perez -
r14990:07072a49 merge
parent child Browse files
Show More
@@ -0,0 +1,6 b''
1 * :class:`IPython.core.inputsplitter.IPythonInputSplitter` no longer has a method
2 ``source_raw_reset()``, but gains :meth:`~IPython.core.inputsplitter.IPythonInputSplitter.raw_reset`
3 instead. Use of ``source_raw_reset`` can be replaced with::
4
5 raw = isp.source_raw
6 transformed = isp.source_reset()
@@ -0,0 +1,4 b''
1 * Input transformers (see :doc:`/config/inputtransforms`) may now raise
2 :exc:`SyntaxError` if they determine that input is invalid. The input
3 transformation machinery in IPython will handle displaying the exception to
4 the user and resetting state.
@@ -501,8 +501,14 b' class IPythonInputSplitter(InputSplitter):'
501 self.source_raw = ''
501 self.source_raw = ''
502 self.transformer_accumulating = False
502 self.transformer_accumulating = False
503 self.within_python_line = False
503 self.within_python_line = False
504
504 for t in self.transforms:
505 for t in self.transforms:
506 try:
505 t.reset()
507 t.reset()
508 except SyntaxError:
509 # Nothing that calls reset() expects to handle transformer
510 # errors
511 pass
506
512
507 def flush_transformers(self):
513 def flush_transformers(self):
508 def _flush(transform, out):
514 def _flush(transform, out):
@@ -519,18 +525,19 b' class IPythonInputSplitter(InputSplitter):'
519 if out is not None:
525 if out is not None:
520 self._store(out)
526 self._store(out)
521
527
522 def source_raw_reset(self):
528 def raw_reset(self):
523 """Return input and raw source and perform a full reset.
529 """Return raw input only and perform a full reset.
524 """
530 """
525 self.flush_transformers()
531 out = self.source_raw
526 out = self.source
527 out_r = self.source_raw
528 self.reset()
532 self.reset()
529 return out, out_r
533 return out
530
534
531 def source_reset(self):
535 def source_reset(self):
536 try:
532 self.flush_transformers()
537 self.flush_transformers()
533 return super(IPythonInputSplitter, self).source_reset()
538 return self.source
539 finally:
540 self.reset()
534
541
535 def push_accepts_more(self):
542 def push_accepts_more(self):
536 if self.transformer_accumulating:
543 if self.transformer_accumulating:
@@ -542,8 +549,12 b' class IPythonInputSplitter(InputSplitter):'
542 """Process and translate a cell of input.
549 """Process and translate a cell of input.
543 """
550 """
544 self.reset()
551 self.reset()
552 try:
545 self.push(cell)
553 self.push(cell)
546 return self.source_reset()
554 self.flush_transformers()
555 return self.source
556 finally:
557 self.reset()
547
558
548 def push(self, lines):
559 def push(self, lines):
549 """Push one or more lines of IPython input.
560 """Push one or more lines of IPython input.
@@ -52,6 +52,9 b' class InputTransformer(with_metaclass(abc.ABCMeta, object)):'
52 input or None if the transformer is waiting for more input.
52 input or None if the transformer is waiting for more input.
53
53
54 Must be overridden by subclasses.
54 Must be overridden by subclasses.
55
56 Implementations may raise ``SyntaxError`` if the input is invalid. No
57 other exceptions may be raised.
55 """
58 """
56 pass
59 pass
57
60
@@ -2649,28 +2649,28 b' class InteractiveShell(SingletonConfigurable):'
2649 if silent:
2649 if silent:
2650 store_history = False
2650 store_history = False
2651
2651
2652 self.input_transformer_manager.push(raw_cell)
2652 # If any of our input transformation (input_transformer_manager or
2653 cell = self.input_transformer_manager.source_reset()
2653 # prefilter_manager) raises an exception, we store it in this variable
2654
2654 # so that we can display the error after logging the input and storing
2655 # Our own compiler remembers the __future__ environment. If we want to
2655 # it in the history.
2656 # run code with a separate __future__ environment, use the default
2656 preprocessing_exc_tuple = None
2657 # compiler
2657 try:
2658 compiler = self.compile if shell_futures else CachingCompiler()
2658 # Static input transformations
2659
2659 cell = self.input_transformer_manager.transform_cell(raw_cell)
2660 with self.builtin_trap:
2660 except SyntaxError:
2661 prefilter_failed = False
2661 preprocessing_exc_tuple = sys.exc_info()
2662 cell = raw_cell # cell has to exist so it can be stored/logged
2663 else:
2662 if len(cell.splitlines()) == 1:
2664 if len(cell.splitlines()) == 1:
2665 # Dynamic transformations - only applied for single line commands
2666 with self.builtin_trap:
2663 try:
2667 try:
2664 # use prefilter_lines to handle trailing newlines
2668 # use prefilter_lines to handle trailing newlines
2665 # restore trailing newline for ast.parse
2669 # restore trailing newline for ast.parse
2666 cell = self.prefilter_manager.prefilter_lines(cell) + '\n'
2670 cell = self.prefilter_manager.prefilter_lines(cell) + '\n'
2667 except AliasError as e:
2668 error(e)
2669 prefilter_failed = True
2670 except Exception:
2671 except Exception:
2671 # don't allow prefilter errors to crash IPython
2672 # don't allow prefilter errors to crash IPython
2672 self.showtraceback()
2673 preprocessing_exc_tuple = sys.exc_info()
2673 prefilter_failed = True
2674
2674
2675 # Store raw and processed history
2675 # Store raw and processed history
2676 if store_history:
2676 if store_history:
@@ -2679,11 +2679,23 b' class InteractiveShell(SingletonConfigurable):'
2679 if not silent:
2679 if not silent:
2680 self.logger.log(cell, raw_cell)
2680 self.logger.log(cell, raw_cell)
2681
2681
2682 if not prefilter_failed:
2682 # Display the exception if input processing failed.
2683 # don't run if prefilter failed
2683 if preprocessing_exc_tuple is not None:
2684 self.showtraceback(preprocessing_exc_tuple)
2685 if store_history:
2686 self.execution_count += 1
2687 return
2688
2689 # Our own compiler remembers the __future__ environment. If we want to
2690 # run code with a separate __future__ environment, use the default
2691 # compiler
2692 compiler = self.compile if shell_futures else CachingCompiler()
2693
2694 with self.builtin_trap:
2684 cell_name = self.compile.cache(cell, self.execution_count)
2695 cell_name = self.compile.cache(cell, self.execution_count)
2685
2696
2686 with self.display_trap:
2697 with self.display_trap:
2698 # Compile to bytecode
2687 try:
2699 try:
2688 code_ast = compiler.ast_parse(cell, filename=cell_name)
2700 code_ast = compiler.ast_parse(cell, filename=cell_name)
2689 except IndentationError:
2701 except IndentationError:
@@ -2698,8 +2710,10 b' class InteractiveShell(SingletonConfigurable):'
2698 self.execution_count += 1
2710 self.execution_count += 1
2699 return None
2711 return None
2700
2712
2713 # Apply AST transformations
2701 code_ast = self.transform_ast(code_ast)
2714 code_ast = self.transform_ast(code_ast)
2702
2715
2716 # Execute the user code
2703 interactivity = "none" if silent else self.ast_node_interactivity
2717 interactivity = "none" if silent else self.ast_node_interactivity
2704 self.run_ast_nodes(code_ast.body, cell_name,
2718 self.run_ast_nodes(code_ast.body, cell_name,
2705 interactivity=interactivity, compiler=compiler)
2719 interactivity=interactivity, compiler=compiler)
@@ -412,7 +412,8 b' class IPythonInputTestCase(InputSplitterTestCase):'
412 continue
412 continue
413
413
414 isp.push(raw+'\n')
414 isp.push(raw+'\n')
415 out, out_raw = isp.source_raw_reset()
415 out_raw = isp.source_raw
416 out = isp.source_reset()
416 self.assertEqual(out.rstrip(), out_t,
417 self.assertEqual(out.rstrip(), out_t,
417 tt.pair_fail_msg.format("inputsplitter",raw, out_t, out))
418 tt.pair_fail_msg.format("inputsplitter",raw, out_t, out))
418 self.assertEqual(out_raw.rstrip(), raw.rstrip())
419 self.assertEqual(out_raw.rstrip(), raw.rstrip())
@@ -431,7 +432,8 b' class IPythonInputTestCase(InputSplitterTestCase):'
431 isp.push(lraw)
432 isp.push(lraw)
432 raw_parts.append(lraw)
433 raw_parts.append(lraw)
433
434
434 out, out_raw = isp.source_raw_reset()
435 out_raw = isp.source_raw
436 out = isp.source_reset()
435 out_t = '\n'.join(out_t_parts).rstrip()
437 out_t = '\n'.join(out_t_parts).rstrip()
436 raw = '\n'.join(raw_parts).rstrip()
438 raw = '\n'.join(raw_parts).rstrip()
437 self.assertEqual(out.rstrip(), out_t)
439 self.assertEqual(out.rstrip(), out_t)
@@ -498,7 +500,8 b" if __name__ == '__main__':"
498 # Here we just return input so we can use it in a test suite, but a
500 # Here we just return input so we can use it in a test suite, but a
499 # real interpreter would instead send it for execution somewhere.
501 # real interpreter would instead send it for execution somewhere.
500 #src = isp.source; raise EOFError # dbg
502 #src = isp.source; raise EOFError # dbg
501 src, raw = isp.source_raw_reset()
503 raw = isp.source_raw
504 src = isp.source_reset()
502 print('Input source was:\n', src)
505 print('Input source was:\n', src)
503 print('Raw source was:\n', raw)
506 print('Raw source was:\n', raw)
504 except EOFError:
507 except EOFError:
@@ -545,9 +548,7 b' class CellMagicsCommon(object):'
545
548
546 def test_whole_cell(self):
549 def test_whole_cell(self):
547 src = "%%cellm line\nbody\n"
550 src = "%%cellm line\nbody\n"
548 sp = self.sp
551 out = self.sp.transform_cell(src)
549 sp.push(src)
550 out = sp.source_reset()
551 ref = u"get_ipython().run_cell_magic({u}'cellm', {u}'line', {u}'body')\n"
552 ref = u"get_ipython().run_cell_magic({u}'cellm', {u}'line', {u}'body')\n"
552 nt.assert_equal(out, py3compat.u_format(ref))
553 nt.assert_equal(out, py3compat.u_format(ref))
553
554
@@ -33,6 +33,7 b' from os.path import join'
33 import nose.tools as nt
33 import nose.tools as nt
34
34
35 # Our own
35 # Our own
36 from IPython.core.inputtransformer import InputTransformer
36 from IPython.testing.decorators import skipif, skip_win32, onlyif_unicode_paths
37 from IPython.testing.decorators import skipif, skip_win32, onlyif_unicode_paths
37 from IPython.testing import tools as tt
38 from IPython.testing import tools as tt
38 from IPython.utils import io
39 from IPython.utils import io
@@ -674,4 +675,41 b' def test_user_expression():'
674
675
675
676
676
677
678 class TestSyntaxErrorTransformer(unittest.TestCase):
679 """Check that SyntaxError raised by an input transformer is handled by run_cell()"""
680
681 class SyntaxErrorTransformer(InputTransformer):
682
683 def push(self, line):
684 pos = line.find('syntaxerror')
685 if pos >= 0:
686 e = SyntaxError('input contains "syntaxerror"')
687 e.text = line
688 e.offset = pos + 1
689 raise e
690 return line
691
692 def reset(self):
693 pass
694
695 def setUp(self):
696 self.transformer = TestSyntaxErrorTransformer.SyntaxErrorTransformer()
697 ip.input_splitter.python_line_transforms.append(self.transformer)
698 ip.input_transformer_manager.python_line_transforms.append(self.transformer)
699
700 def tearDown(self):
701 ip.input_splitter.python_line_transforms.remove(self.transformer)
702 ip.input_transformer_manager.python_line_transforms.remove(self.transformer)
703
704 def test_syntaxerror_input_transformer(self):
705 with tt.AssertPrints('1234'):
706 ip.run_cell('1234')
707 with tt.AssertPrints('SyntaxError: invalid syntax'):
708 ip.run_cell('1 2 3') # plain python syntax error
709 with tt.AssertPrints('SyntaxError: input contains "syntaxerror"'):
710 ip.run_cell('2345 # syntaxerror') # input transformer syntax error
711 with tt.AssertPrints('3456'):
712 ip.run_cell('3456')
713
714
677
715
@@ -224,7 +224,10 b' class FrontendWidget(HistoryConsoleWidget, BaseFrontendMixin):'
224 'interactive' is True; otherwise, it is False.
224 'interactive' is True; otherwise, it is False.
225 """
225 """
226 self._input_splitter.reset()
226 self._input_splitter.reset()
227 try:
227 complete = self._input_splitter.push(source)
228 complete = self._input_splitter.push(source)
229 except SyntaxError:
230 return True
228 if interactive:
231 if interactive:
229 complete = not self._input_splitter.push_accepts_more()
232 complete = not self._input_splitter.push_accepts_more()
230 return complete
233 return complete
@@ -343,7 +343,7 b' class EmbeddedSphinxShell(object):'
343 splitter.push(line)
343 splitter.push(line)
344 more = splitter.push_accepts_more()
344 more = splitter.push_accepts_more()
345 if not more:
345 if not more:
346 source_raw = splitter.source_raw_reset()[1]
346 source_raw = splitter.raw_reset()
347 self.IP.run_cell(source_raw, store_history=store_history)
347 self.IP.run_cell(source_raw, store_history=store_history)
348 finally:
348 finally:
349 sys.stdout = stdout
349 sys.stdout = stdout
@@ -464,7 +464,7 b' class ZMQTerminalInteractiveShell(TerminalInteractiveShell):'
464 #double-guard against keyboardinterrupts during kbdint handling
464 #double-guard against keyboardinterrupts during kbdint handling
465 try:
465 try:
466 self.write('\nKeyboardInterrupt\n')
466 self.write('\nKeyboardInterrupt\n')
467 source_raw = self.input_splitter.source_raw_reset()[1]
467 source_raw = self.input_splitter.raw_reset()
468 hlen_b4_cell = self._replace_rlhist_multiline(source_raw, hlen_b4_cell)
468 hlen_b4_cell = self._replace_rlhist_multiline(source_raw, hlen_b4_cell)
469 more = False
469 more = False
470 except KeyboardInterrupt:
470 except KeyboardInterrupt:
@@ -486,13 +486,18 b' class ZMQTerminalInteractiveShell(TerminalInteractiveShell):'
486 # asynchronously by signal handlers, for example.
486 # asynchronously by signal handlers, for example.
487 self.showtraceback()
487 self.showtraceback()
488 else:
488 else:
489 try:
489 self.input_splitter.push(line)
490 self.input_splitter.push(line)
490 more = self.input_splitter.push_accepts_more()
491 more = self.input_splitter.push_accepts_more()
492 except SyntaxError:
493 # Run the code directly - run_cell takes care of displaying
494 # the exception.
495 more = False
491 if (self.SyntaxTB.last_syntax_error and
496 if (self.SyntaxTB.last_syntax_error and
492 self.autoedit_syntax):
497 self.autoedit_syntax):
493 self.edit_syntax_error()
498 self.edit_syntax_error()
494 if not more:
499 if not more:
495 source_raw = self.input_splitter.source_raw_reset()[1]
500 source_raw = self.input_splitter.raw_reset()
496 hlen_b4_cell = self._replace_rlhist_multiline(source_raw, hlen_b4_cell)
501 hlen_b4_cell = self._replace_rlhist_multiline(source_raw, hlen_b4_cell)
497 self.run_cell(source_raw)
502 self.run_cell(source_raw)
498
503
@@ -529,7 +529,7 b' class TerminalInteractiveShell(InteractiveShell):'
529 #double-guard against keyboardinterrupts during kbdint handling
529 #double-guard against keyboardinterrupts during kbdint handling
530 try:
530 try:
531 self.write('\nKeyboardInterrupt\n')
531 self.write('\nKeyboardInterrupt\n')
532 source_raw = self.input_splitter.source_raw_reset()[1]
532 source_raw = self.input_splitter.raw_reset()
533 hlen_b4_cell = \
533 hlen_b4_cell = \
534 self._replace_rlhist_multiline(source_raw, hlen_b4_cell)
534 self._replace_rlhist_multiline(source_raw, hlen_b4_cell)
535 more = False
535 more = False
@@ -552,13 +552,18 b' class TerminalInteractiveShell(InteractiveShell):'
552 # asynchronously by signal handlers, for example.
552 # asynchronously by signal handlers, for example.
553 self.showtraceback()
553 self.showtraceback()
554 else:
554 else:
555 try:
555 self.input_splitter.push(line)
556 self.input_splitter.push(line)
556 more = self.input_splitter.push_accepts_more()
557 more = self.input_splitter.push_accepts_more()
558 except SyntaxError:
559 # Run the code directly - run_cell takes care of displaying
560 # the exception.
561 more = False
557 if (self.SyntaxTB.last_syntax_error and
562 if (self.SyntaxTB.last_syntax_error and
558 self.autoedit_syntax):
563 self.autoedit_syntax):
559 self.edit_syntax_error()
564 self.edit_syntax_error()
560 if not more:
565 if not more:
561 source_raw = self.input_splitter.source_raw_reset()[1]
566 source_raw = self.input_splitter.raw_reset()
562 self.run_cell(source_raw, store_history=True)
567 self.run_cell(source_raw, store_history=True)
563 hlen_b4_cell = \
568 hlen_b4_cell = \
564 self._replace_rlhist_multiline(source_raw, hlen_b4_cell)
569 self._replace_rlhist_multiline(source_raw, hlen_b4_cell)
@@ -17,12 +17,68 b' Authors'
17 #-----------------------------------------------------------------------------
17 #-----------------------------------------------------------------------------
18 # stdlib
18 # stdlib
19 import sys
19 import sys
20 import types
20 import unittest
21 import unittest
21
22
23 from IPython.core.inputtransformer import InputTransformer
22 from IPython.testing.decorators import skipif
24 from IPython.testing.decorators import skipif
23 from IPython.utils import py3compat
25 from IPython.utils import py3compat
24 from IPython.testing import tools as tt
26 from IPython.testing import tools as tt
25
27
28 # Decorator for interaction loop tests -----------------------------------------
29
30 class mock_input_helper(object):
31 """Machinery for tests of the main interact loop.
32
33 Used by the mock_input decorator.
34 """
35 def __init__(self, testgen):
36 self.testgen = testgen
37 self.exception = None
38 self.ip = get_ipython()
39
40 def __enter__(self):
41 self.orig_raw_input = self.ip.raw_input
42 self.ip.raw_input = self.fake_input
43 return self
44
45 def __exit__(self, etype, value, tb):
46 self.ip.raw_input = self.orig_raw_input
47
48 def fake_input(self, prompt):
49 try:
50 return next(self.testgen)
51 except StopIteration:
52 self.ip.exit_now = True
53 return u''
54 except:
55 self.exception = sys.exc_info()
56 self.ip.exit_now = True
57 return u''
58
59 def mock_input(testfunc):
60 """Decorator for tests of the main interact loop.
61
62 Write the test as a generator, yield-ing the input strings, which IPython
63 will see as if they were typed in at the prompt.
64 """
65 def test_method(self):
66 testgen = testfunc(self)
67 with mock_input_helper(testgen) as mih:
68 mih.ip.interact(display_banner=False)
69
70 if mih.exception is not None:
71 # Re-raise captured exception
72 etype, value, tb = mih.exception
73 import traceback
74 traceback.print_tb(tb, file=sys.stdout)
75 del tb # Avoid reference loop
76 raise value
77
78 return test_method
79
80 # Test classes -----------------------------------------------------------------
81
26 class InteractiveShellTestCase(unittest.TestCase):
82 class InteractiveShellTestCase(unittest.TestCase):
27 def rl_hist_entries(self, rl, n):
83 def rl_hist_entries(self, rl, n):
28 """Get last n readline history entries as a list"""
84 """Get last n readline history entries as a list"""
@@ -171,6 +227,42 b' class InteractiveShellTestCase(unittest.TestCase):'
171 expected = [ py3compat.unicode_to_str(e, enc) for e in expected ]
227 expected = [ py3compat.unicode_to_str(e, enc) for e in expected ]
172 self.assertEqual(hist, expected)
228 self.assertEqual(hist, expected)
173
229
230 @mock_input
231 def test_inputtransformer_syntaxerror(self):
232 ip = get_ipython()
233 transformer = SyntaxErrorTransformer()
234 ip.input_splitter.python_line_transforms.append(transformer)
235 ip.input_transformer_manager.python_line_transforms.append(transformer)
236
237 try:
238 #raise Exception
239 with tt.AssertPrints('4', suppress=False):
240 yield u'print(2*2)'
241
242 with tt.AssertPrints('SyntaxError: input contains', suppress=False):
243 yield u'print(2345) # syntaxerror'
244
245 with tt.AssertPrints('16', suppress=False):
246 yield u'print(4*4)'
247
248 finally:
249 ip.input_splitter.python_line_transforms.remove(transformer)
250 ip.input_transformer_manager.python_line_transforms.remove(transformer)
251
252
253 class SyntaxErrorTransformer(InputTransformer):
254 def push(self, line):
255 pos = line.find('syntaxerror')
256 if pos >= 0:
257 e = SyntaxError('input contains "syntaxerror"')
258 e.text = line
259 e.offset = pos + 1
260 raise e
261 return line
262
263 def reset(self):
264 pass
265
174 class TerminalMagicsTestCase(unittest.TestCase):
266 class TerminalMagicsTestCase(unittest.TestCase):
175 def test_paste_magics_message(self):
267 def test_paste_magics_message(self):
176 """Test that an IndentationError while using paste magics doesn't
268 """Test that an IndentationError while using paste magics doesn't
@@ -44,6 +44,11 b' class IOStream:'
44 for meth in filter(clone, dir(stream)):
44 for meth in filter(clone, dir(stream)):
45 setattr(self, meth, getattr(stream, meth))
45 setattr(self, meth, getattr(stream, meth))
46
46
47 def __repr__(self):
48 cls = self.__class__
49 tpl = '{mod}.{cls}({args})'
50 return tpl.format(mod=cls.__module__, cls=cls.__name__, args=self.stream)
51
47 def write(self,data):
52 def write(self,data):
48 try:
53 try:
49 self._swrite(data)
54 self._swrite(data)
@@ -46,6 +46,14 b' it gets added to both, e.g.::'
46 ip.input_splitter.logical_line_transforms.append(my_transformer())
46 ip.input_splitter.logical_line_transforms.append(my_transformer())
47 ip.input_transformer_manager.logical_line_transforms.append(my_transformer())
47 ip.input_transformer_manager.logical_line_transforms.append(my_transformer())
48
48
49 These transformers may raise :exc:`SyntaxError` if the input code is invalid, but
50 in most cases it is clearer to pass unrecognised code through unmodified and let
51 Python's own parser decide whether it is valid.
52
53 .. versionchanged:: 2.0
54
55 Added the option to raise :exc:`SyntaxError`.
56
49 Stateless transformations
57 Stateless transformations
50 -------------------------
58 -------------------------
51
59
General Comments 0
You need to be logged in to leave comments. Login now