##// END OF EJS Templates
Fix test failure in IPython.lib
Thomas Kluyver -
Show More
@@ -1,661 +1,658 b''
1 """Analysis of text input into executable blocks.
1 """Analysis of text input into executable blocks.
2
2
3 The main class in this module, :class:`InputSplitter`, is designed to break
3 The main class in this module, :class:`InputSplitter`, is designed to break
4 input from either interactive, line-by-line environments or block-based ones,
4 input from either interactive, line-by-line environments or block-based ones,
5 into standalone blocks that can be executed by Python as 'single' statements
5 into standalone blocks that can be executed by Python as 'single' statements
6 (thus triggering sys.displayhook).
6 (thus triggering sys.displayhook).
7
7
8 A companion, :class:`IPythonInputSplitter`, provides the same functionality but
8 A companion, :class:`IPythonInputSplitter`, provides the same functionality but
9 with full support for the extended IPython syntax (magics, system calls, etc).
9 with full support for the extended IPython syntax (magics, system calls, etc).
10
10
11 For more details, see the class docstring below.
11 For more details, see the class docstring below.
12
12
13 Syntax Transformations
13 Syntax Transformations
14 ----------------------
14 ----------------------
15
15
16 One of the main jobs of the code in this file is to apply all syntax
16 One of the main jobs of the code in this file is to apply all syntax
17 transformations that make up 'the IPython language', i.e. magics, shell
17 transformations that make up 'the IPython language', i.e. magics, shell
18 escapes, etc. All transformations should be implemented as *fully stateless*
18 escapes, etc. All transformations should be implemented as *fully stateless*
19 entities, that simply take one line as their input and return a line.
19 entities, that simply take one line as their input and return a line.
20 Internally for implementation purposes they may be a normal function or a
20 Internally for implementation purposes they may be a normal function or a
21 callable object, but the only input they receive will be a single line and they
21 callable object, but the only input they receive will be a single line and they
22 should only return a line, without holding any data-dependent state between
22 should only return a line, without holding any data-dependent state between
23 calls.
23 calls.
24
24
25 As an example, the EscapedTransformer is a class so we can more clearly group
25 As an example, the EscapedTransformer is a class so we can more clearly group
26 together the functionality of dispatching to individual functions based on the
26 together the functionality of dispatching to individual functions based on the
27 starting escape character, but the only method for public use is its call
27 starting escape character, but the only method for public use is its call
28 method.
28 method.
29
29
30
30
31 ToDo
31 ToDo
32 ----
32 ----
33
33
34 - Should we make push() actually raise an exception once push_accepts_more()
34 - Should we make push() actually raise an exception once push_accepts_more()
35 returns False?
35 returns False?
36
36
37 - Naming cleanups. The tr_* names aren't the most elegant, though now they are
37 - Naming cleanups. The tr_* names aren't the most elegant, though now they are
38 at least just attributes of a class so not really very exposed.
38 at least just attributes of a class so not really very exposed.
39
39
40 - Think about the best way to support dynamic things: automagic, autocall,
40 - Think about the best way to support dynamic things: automagic, autocall,
41 macros, etc.
41 macros, etc.
42
42
43 - Think of a better heuristic for the application of the transforms in
43 - Think of a better heuristic for the application of the transforms in
44 IPythonInputSplitter.push() than looking at the buffer ending in ':'. Idea:
44 IPythonInputSplitter.push() than looking at the buffer ending in ':'. Idea:
45 track indentation change events (indent, dedent, nothing) and apply them only
45 track indentation change events (indent, dedent, nothing) and apply them only
46 if the indentation went up, but not otherwise.
46 if the indentation went up, but not otherwise.
47
47
48 - Think of the cleanest way for supporting user-specified transformations (the
48 - Think of the cleanest way for supporting user-specified transformations (the
49 user prefilters we had before).
49 user prefilters we had before).
50
50
51 Authors
51 Authors
52 -------
52 -------
53
53
54 * Fernando Perez
54 * Fernando Perez
55 * Brian Granger
55 * Brian Granger
56 """
56 """
57 #-----------------------------------------------------------------------------
57 #-----------------------------------------------------------------------------
58 # Copyright (C) 2010 The IPython Development Team
58 # Copyright (C) 2010 The IPython Development Team
59 #
59 #
60 # Distributed under the terms of the BSD License. The full license is in
60 # Distributed under the terms of the BSD License. The full license is in
61 # the file COPYING, distributed as part of this software.
61 # the file COPYING, distributed as part of this software.
62 #-----------------------------------------------------------------------------
62 #-----------------------------------------------------------------------------
63
63
64 #-----------------------------------------------------------------------------
64 #-----------------------------------------------------------------------------
65 # Imports
65 # Imports
66 #-----------------------------------------------------------------------------
66 #-----------------------------------------------------------------------------
67 # stdlib
67 # stdlib
68 import ast
68 import ast
69 import codeop
69 import codeop
70 import re
70 import re
71 import sys
71 import sys
72
72
73 # IPython modules
73 # IPython modules
74 from IPython.core.splitinput import split_user_input, LineInfo
74 from IPython.core.splitinput import split_user_input, LineInfo
75 from IPython.utils.py3compat import cast_unicode
75 from IPython.utils.py3compat import cast_unicode
76 from IPython.core.inputtransformer import (leading_indent,
76 from IPython.core.inputtransformer import (leading_indent,
77 classic_prompt,
77 classic_prompt,
78 ipy_prompt,
78 ipy_prompt,
79 cellmagic,
79 cellmagic,
80 assemble_logical_lines,
80 assemble_logical_lines,
81 help_end,
81 help_end,
82 escaped_commands,
82 escaped_commands,
83 assign_from_magic,
83 assign_from_magic,
84 assign_from_system,
84 assign_from_system,
85 assemble_python_lines,
85 assemble_python_lines,
86 )
86 )
87
87
88 # Temporary!
88 # Temporary!
89 from IPython.core.inputtransformer import (ESC_SHELL, ESC_SH_CAP, ESC_HELP,
89 from IPython.core.inputtransformer import (ESC_SHELL, ESC_SH_CAP, ESC_HELP,
90 ESC_HELP2, ESC_MAGIC, ESC_MAGIC2,
90 ESC_HELP2, ESC_MAGIC, ESC_MAGIC2,
91 ESC_QUOTE, ESC_QUOTE2, ESC_PAREN, ESC_SEQUENCES)
91 ESC_QUOTE, ESC_QUOTE2, ESC_PAREN, ESC_SEQUENCES)
92
92
93 #-----------------------------------------------------------------------------
93 #-----------------------------------------------------------------------------
94 # Utilities
94 # Utilities
95 #-----------------------------------------------------------------------------
95 #-----------------------------------------------------------------------------
96
96
97 # FIXME: These are general-purpose utilities that later can be moved to the
97 # FIXME: These are general-purpose utilities that later can be moved to the
98 # general ward. Kept here for now because we're being very strict about test
98 # general ward. Kept here for now because we're being very strict about test
99 # coverage with this code, and this lets us ensure that we keep 100% coverage
99 # coverage with this code, and this lets us ensure that we keep 100% coverage
100 # while developing.
100 # while developing.
101
101
102 # compiled regexps for autoindent management
102 # compiled regexps for autoindent management
103 dedent_re = re.compile('|'.join([
103 dedent_re = re.compile('|'.join([
104 r'^\s+raise(\s.*)?$', # raise statement (+ space + other stuff, maybe)
104 r'^\s+raise(\s.*)?$', # raise statement (+ space + other stuff, maybe)
105 r'^\s+raise\([^\)]*\).*$', # wacky raise with immediate open paren
105 r'^\s+raise\([^\)]*\).*$', # wacky raise with immediate open paren
106 r'^\s+return(\s.*)?$', # normal return (+ space + other stuff, maybe)
106 r'^\s+return(\s.*)?$', # normal return (+ space + other stuff, maybe)
107 r'^\s+return\([^\)]*\).*$', # wacky return with immediate open paren
107 r'^\s+return\([^\)]*\).*$', # wacky return with immediate open paren
108 r'^\s+pass\s*$', # pass (optionally followed by trailing spaces)
108 r'^\s+pass\s*$', # pass (optionally followed by trailing spaces)
109 r'^\s+break\s*$', # break (optionally followed by trailing spaces)
109 r'^\s+break\s*$', # break (optionally followed by trailing spaces)
110 r'^\s+continue\s*$', # continue (optionally followed by trailing spaces)
110 r'^\s+continue\s*$', # continue (optionally followed by trailing spaces)
111 ]))
111 ]))
112 ini_spaces_re = re.compile(r'^([ \t\r\f\v]+)')
112 ini_spaces_re = re.compile(r'^([ \t\r\f\v]+)')
113
113
114 # regexp to match pure comment lines so we don't accidentally insert 'if 1:'
114 # regexp to match pure comment lines so we don't accidentally insert 'if 1:'
115 # before pure comments
115 # before pure comments
116 comment_line_re = re.compile('^\s*\#')
116 comment_line_re = re.compile('^\s*\#')
117
117
118
118
119 def num_ini_spaces(s):
119 def num_ini_spaces(s):
120 """Return the number of initial spaces in a string.
120 """Return the number of initial spaces in a string.
121
121
122 Note that tabs are counted as a single space. For now, we do *not* support
122 Note that tabs are counted as a single space. For now, we do *not* support
123 mixing of tabs and spaces in the user's input.
123 mixing of tabs and spaces in the user's input.
124
124
125 Parameters
125 Parameters
126 ----------
126 ----------
127 s : string
127 s : string
128
128
129 Returns
129 Returns
130 -------
130 -------
131 n : int
131 n : int
132 """
132 """
133
133
134 ini_spaces = ini_spaces_re.match(s)
134 ini_spaces = ini_spaces_re.match(s)
135 if ini_spaces:
135 if ini_spaces:
136 return ini_spaces.end()
136 return ini_spaces.end()
137 else:
137 else:
138 return 0
138 return 0
139
139
140 def last_blank(src):
140 def last_blank(src):
141 """Determine if the input source ends in a blank.
141 """Determine if the input source ends in a blank.
142
142
143 A blank is either a newline or a line consisting of whitespace.
143 A blank is either a newline or a line consisting of whitespace.
144
144
145 Parameters
145 Parameters
146 ----------
146 ----------
147 src : string
147 src : string
148 A single or multiline string.
148 A single or multiline string.
149 """
149 """
150 if not src: return False
150 if not src: return False
151 ll = src.splitlines()[-1]
151 ll = src.splitlines()[-1]
152 return (ll == '') or ll.isspace()
152 return (ll == '') or ll.isspace()
153
153
154
154
155 last_two_blanks_re = re.compile(r'\n\s*\n\s*$', re.MULTILINE)
155 last_two_blanks_re = re.compile(r'\n\s*\n\s*$', re.MULTILINE)
156 last_two_blanks_re2 = re.compile(r'.+\n\s*\n\s+$', re.MULTILINE)
156 last_two_blanks_re2 = re.compile(r'.+\n\s*\n\s+$', re.MULTILINE)
157
157
158 def last_two_blanks(src):
158 def last_two_blanks(src):
159 """Determine if the input source ends in two blanks.
159 """Determine if the input source ends in two blanks.
160
160
161 A blank is either a newline or a line consisting of whitespace.
161 A blank is either a newline or a line consisting of whitespace.
162
162
163 Parameters
163 Parameters
164 ----------
164 ----------
165 src : string
165 src : string
166 A single or multiline string.
166 A single or multiline string.
167 """
167 """
168 if not src: return False
168 if not src: return False
169 # The logic here is tricky: I couldn't get a regexp to work and pass all
169 # The logic here is tricky: I couldn't get a regexp to work and pass all
170 # the tests, so I took a different approach: split the source by lines,
170 # the tests, so I took a different approach: split the source by lines,
171 # grab the last two and prepend '###\n' as a stand-in for whatever was in
171 # grab the last two and prepend '###\n' as a stand-in for whatever was in
172 # the body before the last two lines. Then, with that structure, it's
172 # the body before the last two lines. Then, with that structure, it's
173 # possible to analyze with two regexps. Not the most elegant solution, but
173 # possible to analyze with two regexps. Not the most elegant solution, but
174 # it works. If anyone tries to change this logic, make sure to validate
174 # it works. If anyone tries to change this logic, make sure to validate
175 # the whole test suite first!
175 # the whole test suite first!
176 new_src = '\n'.join(['###\n'] + src.splitlines()[-2:])
176 new_src = '\n'.join(['###\n'] + src.splitlines()[-2:])
177 return (bool(last_two_blanks_re.match(new_src)) or
177 return (bool(last_two_blanks_re.match(new_src)) or
178 bool(last_two_blanks_re2.match(new_src)) )
178 bool(last_two_blanks_re2.match(new_src)) )
179
179
180
180
181 def remove_comments(src):
181 def remove_comments(src):
182 """Remove all comments from input source.
182 """Remove all comments from input source.
183
183
184 Note: comments are NOT recognized inside of strings!
184 Note: comments are NOT recognized inside of strings!
185
185
186 Parameters
186 Parameters
187 ----------
187 ----------
188 src : string
188 src : string
189 A single or multiline input string.
189 A single or multiline input string.
190
190
191 Returns
191 Returns
192 -------
192 -------
193 String with all Python comments removed.
193 String with all Python comments removed.
194 """
194 """
195
195
196 return re.sub('#.*', '', src)
196 return re.sub('#.*', '', src)
197
197
198
198
199 def get_input_encoding():
199 def get_input_encoding():
200 """Return the default standard input encoding.
200 """Return the default standard input encoding.
201
201
202 If sys.stdin has no encoding, 'ascii' is returned."""
202 If sys.stdin has no encoding, 'ascii' is returned."""
203 # There are strange environments for which sys.stdin.encoding is None. We
203 # There are strange environments for which sys.stdin.encoding is None. We
204 # ensure that a valid encoding is returned.
204 # ensure that a valid encoding is returned.
205 encoding = getattr(sys.stdin, 'encoding', None)
205 encoding = getattr(sys.stdin, 'encoding', None)
206 if encoding is None:
206 if encoding is None:
207 encoding = 'ascii'
207 encoding = 'ascii'
208 return encoding
208 return encoding
209
209
210 #-----------------------------------------------------------------------------
210 #-----------------------------------------------------------------------------
211 # Classes and functions for normal Python syntax handling
211 # Classes and functions for normal Python syntax handling
212 #-----------------------------------------------------------------------------
212 #-----------------------------------------------------------------------------
213
213
214 class InputSplitter(object):
214 class InputSplitter(object):
215 """An object that can accumulate lines of Python source before execution.
215 """An object that can accumulate lines of Python source before execution.
216
216
217 This object is designed to be fed python source line-by-line, using
217 This object is designed to be fed python source line-by-line, using
218 :meth:`push`. It will return on each push whether the currently pushed
218 :meth:`push`. It will return on each push whether the currently pushed
219 code could be executed already. In addition, it provides a method called
219 code could be executed already. In addition, it provides a method called
220 :meth:`push_accepts_more` that can be used to query whether more input
220 :meth:`push_accepts_more` that can be used to query whether more input
221 can be pushed into a single interactive block.
221 can be pushed into a single interactive block.
222
222
223 This is a simple example of how an interactive terminal-based client can use
223 This is a simple example of how an interactive terminal-based client can use
224 this tool::
224 this tool::
225
225
226 isp = InputSplitter()
226 isp = InputSplitter()
227 while isp.push_accepts_more():
227 while isp.push_accepts_more():
228 indent = ' '*isp.indent_spaces
228 indent = ' '*isp.indent_spaces
229 prompt = '>>> ' + indent
229 prompt = '>>> ' + indent
230 line = indent + raw_input(prompt)
230 line = indent + raw_input(prompt)
231 isp.push(line)
231 isp.push(line)
232 print 'Input source was:\n', isp.source_reset(),
232 print 'Input source was:\n', isp.source_reset(),
233 """
233 """
234 # Number of spaces of indentation computed from input that has been pushed
234 # Number of spaces of indentation computed from input that has been pushed
235 # so far. This is the attributes callers should query to get the current
235 # so far. This is the attributes callers should query to get the current
236 # indentation level, in order to provide auto-indent facilities.
236 # indentation level, in order to provide auto-indent facilities.
237 indent_spaces = 0
237 indent_spaces = 0
238 # String, indicating the default input encoding. It is computed by default
238 # String, indicating the default input encoding. It is computed by default
239 # at initialization time via get_input_encoding(), but it can be reset by a
239 # at initialization time via get_input_encoding(), but it can be reset by a
240 # client with specific knowledge of the encoding.
240 # client with specific knowledge of the encoding.
241 encoding = ''
241 encoding = ''
242 # String where the current full source input is stored, properly encoded.
242 # String where the current full source input is stored, properly encoded.
243 # Reading this attribute is the normal way of querying the currently pushed
243 # Reading this attribute is the normal way of querying the currently pushed
244 # source code, that has been properly encoded.
244 # source code, that has been properly encoded.
245 source = ''
245 source = ''
246 # Code object corresponding to the current source. It is automatically
246 # Code object corresponding to the current source. It is automatically
247 # synced to the source, so it can be queried at any time to obtain the code
247 # synced to the source, so it can be queried at any time to obtain the code
248 # object; it will be None if the source doesn't compile to valid Python.
248 # object; it will be None if the source doesn't compile to valid Python.
249 code = None
249 code = None
250
250
251 # Private attributes
251 # Private attributes
252
252
253 # List with lines of input accumulated so far
253 # List with lines of input accumulated so far
254 _buffer = None
254 _buffer = None
255 # Command compiler
255 # Command compiler
256 _compile = None
256 _compile = None
257 # Mark when input has changed indentation all the way back to flush-left
257 # Mark when input has changed indentation all the way back to flush-left
258 _full_dedent = False
258 _full_dedent = False
259 # Boolean indicating whether the current block is complete
259 # Boolean indicating whether the current block is complete
260 _is_complete = None
260 _is_complete = None
261
261
262 def __init__(self):
262 def __init__(self):
263 """Create a new InputSplitter instance.
263 """Create a new InputSplitter instance.
264 """
264 """
265 self._buffer = []
265 self._buffer = []
266 self._compile = codeop.CommandCompiler()
266 self._compile = codeop.CommandCompiler()
267 self.encoding = get_input_encoding()
267 self.encoding = get_input_encoding()
268
268
269 def reset(self):
269 def reset(self):
270 """Reset the input buffer and associated state."""
270 """Reset the input buffer and associated state."""
271 self.indent_spaces = 0
271 self.indent_spaces = 0
272 self._buffer[:] = []
272 self._buffer[:] = []
273 self.source = ''
273 self.source = ''
274 self.code = None
274 self.code = None
275 self._is_complete = False
275 self._is_complete = False
276 self._full_dedent = False
276 self._full_dedent = False
277
277
278 def source_reset(self):
278 def source_reset(self):
279 """Return the input source and perform a full reset.
279 """Return the input source and perform a full reset.
280 """
280 """
281 out = self.source
281 out = self.source
282 self.reset()
282 self.reset()
283 return out
283 return out
284
284
285 def push(self, lines):
285 def push(self, lines):
286 """Push one or more lines of input.
286 """Push one or more lines of input.
287
287
288 This stores the given lines and returns a status code indicating
288 This stores the given lines and returns a status code indicating
289 whether the code forms a complete Python block or not.
289 whether the code forms a complete Python block or not.
290
290
291 Any exceptions generated in compilation are swallowed, but if an
291 Any exceptions generated in compilation are swallowed, but if an
292 exception was produced, the method returns True.
292 exception was produced, the method returns True.
293
293
294 Parameters
294 Parameters
295 ----------
295 ----------
296 lines : string
296 lines : string
297 One or more lines of Python input.
297 One or more lines of Python input.
298
298
299 Returns
299 Returns
300 -------
300 -------
301 is_complete : boolean
301 is_complete : boolean
302 True if the current input source (the result of the current input
302 True if the current input source (the result of the current input
303 plus prior inputs) forms a complete Python execution block. Note that
303 plus prior inputs) forms a complete Python execution block. Note that
304 this value is also stored as a private attribute (``_is_complete``), so it
304 this value is also stored as a private attribute (``_is_complete``), so it
305 can be queried at any time.
305 can be queried at any time.
306 """
306 """
307 self._store(lines)
307 self._store(lines)
308 source = self.source
308 source = self.source
309
309
310 # Before calling _compile(), reset the code object to None so that if an
310 # Before calling _compile(), reset the code object to None so that if an
311 # exception is raised in compilation, we don't mislead by having
311 # exception is raised in compilation, we don't mislead by having
312 # inconsistent code/source attributes.
312 # inconsistent code/source attributes.
313 self.code, self._is_complete = None, None
313 self.code, self._is_complete = None, None
314
314
315 # Honor termination lines properly
315 # Honor termination lines properly
316 if source.endswith('\\\n'):
316 if source.endswith('\\\n'):
317 return False
317 return False
318
318
319 self._update_indent(lines)
319 self._update_indent(lines)
320 try:
320 try:
321 self.code = self._compile(source, symbol="exec")
321 self.code = self._compile(source, symbol="exec")
322 # Invalid syntax can produce any of a number of different errors from
322 # Invalid syntax can produce any of a number of different errors from
323 # inside the compiler, so we have to catch them all. Syntax errors
323 # inside the compiler, so we have to catch them all. Syntax errors
324 # immediately produce a 'ready' block, so the invalid Python can be
324 # immediately produce a 'ready' block, so the invalid Python can be
325 # sent to the kernel for evaluation with possible ipython
325 # sent to the kernel for evaluation with possible ipython
326 # special-syntax conversion.
326 # special-syntax conversion.
327 except (SyntaxError, OverflowError, ValueError, TypeError,
327 except (SyntaxError, OverflowError, ValueError, TypeError,
328 MemoryError):
328 MemoryError):
329 self._is_complete = True
329 self._is_complete = True
330 else:
330 else:
331 # Compilation didn't produce any exceptions (though it may not have
331 # Compilation didn't produce any exceptions (though it may not have
332 # given a complete code object)
332 # given a complete code object)
333 self._is_complete = self.code is not None
333 self._is_complete = self.code is not None
334
334
335 return self._is_complete
335 return self._is_complete
336
336
337 def push_accepts_more(self):
337 def push_accepts_more(self):
338 """Return whether a block of interactive input can accept more input.
338 """Return whether a block of interactive input can accept more input.
339
339
340 This method is meant to be used by line-oriented frontends, who need to
340 This method is meant to be used by line-oriented frontends, who need to
341 guess whether a block is complete or not based solely on prior and
341 guess whether a block is complete or not based solely on prior and
342 current input lines. The InputSplitter considers it has a complete
342 current input lines. The InputSplitter considers it has a complete
343 interactive block and will not accept more input only when either a
343 interactive block and will not accept more input when either:
344 SyntaxError is raised, or *all* of the following are true:
345
344
346 1. The input compiles to a complete statement.
345 * A SyntaxError is raised
347
346
348 2. The indentation level is flush-left (because if we are indented,
347 * The code is complete and consists of a single line or a single
349 like inside a function definition or for loop, we need to keep
348 non-compound statement
350 reading new input).
351
349
352 3. There is one extra line consisting only of whitespace.
350 * The code is complete and has a blank line at the end
353
354 Because of condition #3, this method should be used only by
355 *line-oriented* frontends, since it means that intermediate blank lines
356 are not allowed in function definitions (or any other indented block).
357
351
358 If the current input produces a syntax error, this method immediately
352 If the current input produces a syntax error, this method immediately
359 returns False but does *not* raise the syntax error exception, as
353 returns False but does *not* raise the syntax error exception, as
360 typically clients will want to send invalid syntax to an execution
354 typically clients will want to send invalid syntax to an execution
361 backend which might convert the invalid syntax into valid Python via
355 backend which might convert the invalid syntax into valid Python via
362 one of the dynamic IPython mechanisms.
356 one of the dynamic IPython mechanisms.
363 """
357 """
364
358
365 # With incomplete input, unconditionally accept more
359 # With incomplete input, unconditionally accept more
366 # A syntax error also sets _is_complete to True - see push()
360 # A syntax error also sets _is_complete to True - see push()
367 if not self._is_complete:
361 if not self._is_complete:
368 #print("Not complete") # debug
362 #print("Not complete") # debug
369 return True
363 return True
370
364
371 # The user can make any (complete) input execute by leaving a blank line
365 # The user can make any (complete) input execute by leaving a blank line
372 last_line = self.source.splitlines()[-1]
366 last_line = self.source.splitlines()[-1]
373 if (not last_line) or last_line.isspace():
367 if (not last_line) or last_line.isspace():
374 #print("Blank line") # debug
368 #print("Blank line") # debug
375 return False
369 return False
376
370
377 # If there's just a single AST node, and we're flush left, as is the
371 # If there's just a single line or AST node, and we're flush left, as is
378 # case after a simple statement such as 'a=1', we want to execute it
372 # the case after a simple statement such as 'a=1', we want to execute it
379 # straight away.
373 # straight away.
380 if self.indent_spaces==0:
374 if self.indent_spaces==0:
375 if len(self.source.splitlines()) <= 1:
376 return False
377
381 try:
378 try:
382 code_ast = ast.parse(u''.join(self._buffer))
379 code_ast = ast.parse(u''.join(self._buffer))
383 except Exception:
380 except Exception:
384 #print("Can't parse AST") # debug
381 #print("Can't parse AST") # debug
385 return False
382 return False
386 else:
383 else:
387 if len(code_ast.body) == 1 and \
384 if len(code_ast.body) == 1 and \
388 not hasattr(code_ast.body[0], 'body'):
385 not hasattr(code_ast.body[0], 'body'):
389 #print("Simple statement") # debug
386 #print("Simple statement") # debug
390 return False
387 return False
391
388
392 # General fallback - accept more code
389 # General fallback - accept more code
393 return True
390 return True
394
391
395 #------------------------------------------------------------------------
392 #------------------------------------------------------------------------
396 # Private interface
393 # Private interface
397 #------------------------------------------------------------------------
394 #------------------------------------------------------------------------
398
395
399 def _find_indent(self, line):
396 def _find_indent(self, line):
400 """Compute the new indentation level for a single line.
397 """Compute the new indentation level for a single line.
401
398
402 Parameters
399 Parameters
403 ----------
400 ----------
404 line : str
401 line : str
405 A single new line of non-whitespace, non-comment Python input.
402 A single new line of non-whitespace, non-comment Python input.
406
403
407 Returns
404 Returns
408 -------
405 -------
409 indent_spaces : int
406 indent_spaces : int
410 New value for the indent level (it may be equal to self.indent_spaces
407 New value for the indent level (it may be equal to self.indent_spaces
411 if indentation doesn't change.
408 if indentation doesn't change.
412
409
413 full_dedent : boolean
410 full_dedent : boolean
414 Whether the new line causes a full flush-left dedent.
411 Whether the new line causes a full flush-left dedent.
415 """
412 """
416 indent_spaces = self.indent_spaces
413 indent_spaces = self.indent_spaces
417 full_dedent = self._full_dedent
414 full_dedent = self._full_dedent
418
415
419 inisp = num_ini_spaces(line)
416 inisp = num_ini_spaces(line)
420 if inisp < indent_spaces:
417 if inisp < indent_spaces:
421 indent_spaces = inisp
418 indent_spaces = inisp
422 if indent_spaces <= 0:
419 if indent_spaces <= 0:
423 #print 'Full dedent in text',self.source # dbg
420 #print 'Full dedent in text',self.source # dbg
424 full_dedent = True
421 full_dedent = True
425
422
426 if line.rstrip()[-1] == ':':
423 if line.rstrip()[-1] == ':':
427 indent_spaces += 4
424 indent_spaces += 4
428 elif dedent_re.match(line):
425 elif dedent_re.match(line):
429 indent_spaces -= 4
426 indent_spaces -= 4
430 if indent_spaces <= 0:
427 if indent_spaces <= 0:
431 full_dedent = True
428 full_dedent = True
432
429
433 # Safety
430 # Safety
434 if indent_spaces < 0:
431 if indent_spaces < 0:
435 indent_spaces = 0
432 indent_spaces = 0
436 #print 'safety' # dbg
433 #print 'safety' # dbg
437
434
438 return indent_spaces, full_dedent
435 return indent_spaces, full_dedent
439
436
440 def _update_indent(self, lines):
437 def _update_indent(self, lines):
441 for line in remove_comments(lines).splitlines():
438 for line in remove_comments(lines).splitlines():
442 if line and not line.isspace():
439 if line and not line.isspace():
443 self.indent_spaces, self._full_dedent = self._find_indent(line)
440 self.indent_spaces, self._full_dedent = self._find_indent(line)
444
441
445 def _store(self, lines, buffer=None, store='source'):
442 def _store(self, lines, buffer=None, store='source'):
446 """Store one or more lines of input.
443 """Store one or more lines of input.
447
444
448 If input lines are not newline-terminated, a newline is automatically
445 If input lines are not newline-terminated, a newline is automatically
449 appended."""
446 appended."""
450
447
451 if buffer is None:
448 if buffer is None:
452 buffer = self._buffer
449 buffer = self._buffer
453
450
454 if lines.endswith('\n'):
451 if lines.endswith('\n'):
455 buffer.append(lines)
452 buffer.append(lines)
456 else:
453 else:
457 buffer.append(lines+'\n')
454 buffer.append(lines+'\n')
458 setattr(self, store, self._set_source(buffer))
455 setattr(self, store, self._set_source(buffer))
459
456
460 def _set_source(self, buffer):
457 def _set_source(self, buffer):
461 return u''.join(buffer)
458 return u''.join(buffer)
462
459
463
460
464 class IPythonInputSplitter(InputSplitter):
461 class IPythonInputSplitter(InputSplitter):
465 """An input splitter that recognizes all of IPython's special syntax."""
462 """An input splitter that recognizes all of IPython's special syntax."""
466
463
467 # String with raw, untransformed input.
464 # String with raw, untransformed input.
468 source_raw = ''
465 source_raw = ''
469
466
470 # Flag to track when a transformer has stored input that it hasn't given
467 # Flag to track when a transformer has stored input that it hasn't given
471 # back yet.
468 # back yet.
472 transformer_accumulating = False
469 transformer_accumulating = False
473
470
474 # Flag to track when assemble_python_lines has stored input that it hasn't
471 # Flag to track when assemble_python_lines has stored input that it hasn't
475 # given back yet.
472 # given back yet.
476 within_python_line = False
473 within_python_line = False
477
474
478 # Private attributes
475 # Private attributes
479
476
480 # List with lines of raw input accumulated so far.
477 # List with lines of raw input accumulated so far.
481 _buffer_raw = None
478 _buffer_raw = None
482
479
483 def __init__(self, line_input_checker=False, physical_line_transforms=None,
480 def __init__(self, line_input_checker=False, physical_line_transforms=None,
484 logical_line_transforms=None, python_line_transforms=None):
481 logical_line_transforms=None, python_line_transforms=None):
485 super(IPythonInputSplitter, self).__init__()
482 super(IPythonInputSplitter, self).__init__()
486 self._buffer_raw = []
483 self._buffer_raw = []
487 self._validate = True
484 self._validate = True
488
485
489 if physical_line_transforms is not None:
486 if physical_line_transforms is not None:
490 self.physical_line_transforms = physical_line_transforms
487 self.physical_line_transforms = physical_line_transforms
491 else:
488 else:
492 self.physical_line_transforms = [leading_indent(),
489 self.physical_line_transforms = [leading_indent(),
493 classic_prompt(),
490 classic_prompt(),
494 ipy_prompt(),
491 ipy_prompt(),
495 ]
492 ]
496
493
497 self.assemble_logical_lines = assemble_logical_lines()
494 self.assemble_logical_lines = assemble_logical_lines()
498 if logical_line_transforms is not None:
495 if logical_line_transforms is not None:
499 self.logical_line_transforms = logical_line_transforms
496 self.logical_line_transforms = logical_line_transforms
500 else:
497 else:
501 self.logical_line_transforms = [cellmagic(end_on_blank_line=line_input_checker),
498 self.logical_line_transforms = [cellmagic(end_on_blank_line=line_input_checker),
502 help_end(),
499 help_end(),
503 escaped_commands(),
500 escaped_commands(),
504 assign_from_magic(),
501 assign_from_magic(),
505 assign_from_system(),
502 assign_from_system(),
506 ]
503 ]
507
504
508 self.assemble_python_lines = assemble_python_lines()
505 self.assemble_python_lines = assemble_python_lines()
509 if python_line_transforms is not None:
506 if python_line_transforms is not None:
510 self.python_line_transforms = python_line_transforms
507 self.python_line_transforms = python_line_transforms
511 else:
508 else:
512 # We don't use any of these at present
509 # We don't use any of these at present
513 self.python_line_transforms = []
510 self.python_line_transforms = []
514
511
515 @property
512 @property
516 def transforms(self):
513 def transforms(self):
517 "Quick access to all transformers."
514 "Quick access to all transformers."
518 return self.physical_line_transforms + \
515 return self.physical_line_transforms + \
519 [self.assemble_logical_lines] + self.logical_line_transforms + \
516 [self.assemble_logical_lines] + self.logical_line_transforms + \
520 [self.assemble_python_lines] + self.python_line_transforms
517 [self.assemble_python_lines] + self.python_line_transforms
521
518
522 @property
519 @property
523 def transforms_in_use(self):
520 def transforms_in_use(self):
524 """Transformers, excluding logical line transformers if we're in a
521 """Transformers, excluding logical line transformers if we're in a
525 Python line."""
522 Python line."""
526 t = self.physical_line_transforms[:]
523 t = self.physical_line_transforms[:]
527 if not self.within_python_line:
524 if not self.within_python_line:
528 t += [self.assemble_logical_lines] + self.logical_line_transforms
525 t += [self.assemble_logical_lines] + self.logical_line_transforms
529 return t + [self.assemble_python_lines] + self.python_line_transforms
526 return t + [self.assemble_python_lines] + self.python_line_transforms
530
527
531 def reset(self):
528 def reset(self):
532 """Reset the input buffer and associated state."""
529 """Reset the input buffer and associated state."""
533 super(IPythonInputSplitter, self).reset()
530 super(IPythonInputSplitter, self).reset()
534 self._buffer_raw[:] = []
531 self._buffer_raw[:] = []
535 self.source_raw = ''
532 self.source_raw = ''
536 self.transformer_accumulating = False
533 self.transformer_accumulating = False
537 self.within_python_line = False
534 self.within_python_line = False
538 for t in self.transforms:
535 for t in self.transforms:
539 t.reset()
536 t.reset()
540
537
541 def flush_transformers(self):
538 def flush_transformers(self):
542 def _flush(transform, out):
539 def _flush(transform, out):
543 if out is not None:
540 if out is not None:
544 tmp = transform.push(out)
541 tmp = transform.push(out)
545 return tmp or transform.reset() or None
542 return tmp or transform.reset() or None
546 else:
543 else:
547 return transform.reset() or None
544 return transform.reset() or None
548
545
549 out = None
546 out = None
550 for t in self.transforms_in_use:
547 for t in self.transforms_in_use:
551 out = _flush(t, out)
548 out = _flush(t, out)
552
549
553 if out is not None:
550 if out is not None:
554 self._store(out)
551 self._store(out)
555
552
556 def source_raw_reset(self):
553 def source_raw_reset(self):
557 """Return input and raw source and perform a full reset.
554 """Return input and raw source and perform a full reset.
558 """
555 """
559 self.flush_transformers()
556 self.flush_transformers()
560 out = self.source
557 out = self.source
561 out_r = self.source_raw
558 out_r = self.source_raw
562 self.reset()
559 self.reset()
563 return out, out_r
560 return out, out_r
564
561
565 def source_reset(self):
562 def source_reset(self):
566 self.flush_transformers()
563 self.flush_transformers()
567 return super(IPythonInputSplitter, self).source_reset()
564 return super(IPythonInputSplitter, self).source_reset()
568
565
569 def push_accepts_more(self):
566 def push_accepts_more(self):
570 if self.transformer_accumulating:
567 if self.transformer_accumulating:
571 return True
568 return True
572 else:
569 else:
573 return super(IPythonInputSplitter, self).push_accepts_more()
570 return super(IPythonInputSplitter, self).push_accepts_more()
574
571
575 def transform_cell(self, cell):
572 def transform_cell(self, cell):
576 """Process and translate a cell of input.
573 """Process and translate a cell of input.
577 """
574 """
578 self.reset()
575 self.reset()
579 self.push(cell)
576 self.push(cell)
580 return self.source_reset()
577 return self.source_reset()
581
578
582 def push(self, lines):
579 def push(self, lines):
583 """Push one or more lines of IPython input.
580 """Push one or more lines of IPython input.
584
581
585 This stores the given lines and returns a status code indicating
582 This stores the given lines and returns a status code indicating
586 whether the code forms a complete Python block or not, after processing
583 whether the code forms a complete Python block or not, after processing
587 all input lines for special IPython syntax.
584 all input lines for special IPython syntax.
588
585
589 Any exceptions generated in compilation are swallowed, but if an
586 Any exceptions generated in compilation are swallowed, but if an
590 exception was produced, the method returns True.
587 exception was produced, the method returns True.
591
588
592 Parameters
589 Parameters
593 ----------
590 ----------
594 lines : string
591 lines : string
595 One or more lines of Python input.
592 One or more lines of Python input.
596
593
597 Returns
594 Returns
598 -------
595 -------
599 is_complete : boolean
596 is_complete : boolean
600 True if the current input source (the result of the current input
597 True if the current input source (the result of the current input
601 plus prior inputs) forms a complete Python execution block. Note that
598 plus prior inputs) forms a complete Python execution block. Note that
602 this value is also stored as a private attribute (_is_complete), so it
599 this value is also stored as a private attribute (_is_complete), so it
603 can be queried at any time.
600 can be queried at any time.
604 """
601 """
605
602
606 # We must ensure all input is pure unicode
603 # We must ensure all input is pure unicode
607 lines = cast_unicode(lines, self.encoding)
604 lines = cast_unicode(lines, self.encoding)
608
605
609 # ''.splitlines() --> [], but we need to push the empty line to transformers
606 # ''.splitlines() --> [], but we need to push the empty line to transformers
610 lines_list = lines.splitlines()
607 lines_list = lines.splitlines()
611 if not lines_list:
608 if not lines_list:
612 lines_list = ['']
609 lines_list = ['']
613
610
614 # Store raw source before applying any transformations to it. Note
611 # Store raw source before applying any transformations to it. Note
615 # that this must be done *after* the reset() call that would otherwise
612 # that this must be done *after* the reset() call that would otherwise
616 # flush the buffer.
613 # flush the buffer.
617 self._store(lines, self._buffer_raw, 'source_raw')
614 self._store(lines, self._buffer_raw, 'source_raw')
618
615
619 for line in lines_list:
616 for line in lines_list:
620 out = self.push_line(line)
617 out = self.push_line(line)
621
618
622 return out
619 return out
623
620
624 def push_line(self, line):
621 def push_line(self, line):
625 buf = self._buffer
622 buf = self._buffer
626
623
627 def _accumulating(dbg):
624 def _accumulating(dbg):
628 #print(dbg)
625 #print(dbg)
629 self.transformer_accumulating = True
626 self.transformer_accumulating = True
630 return False
627 return False
631
628
632 for transformer in self.physical_line_transforms:
629 for transformer in self.physical_line_transforms:
633 line = transformer.push(line)
630 line = transformer.push(line)
634 if line is None:
631 if line is None:
635 return _accumulating(transformer)
632 return _accumulating(transformer)
636
633
637 if not self.within_python_line:
634 if not self.within_python_line:
638 line = self.assemble_logical_lines.push(line)
635 line = self.assemble_logical_lines.push(line)
639 if line is None:
636 if line is None:
640 return _accumulating('acc logical line')
637 return _accumulating('acc logical line')
641
638
642 for transformer in self.logical_line_transforms:
639 for transformer in self.logical_line_transforms:
643 line = transformer.push(line)
640 line = transformer.push(line)
644 if line is None:
641 if line is None:
645 return _accumulating(transformer)
642 return _accumulating(transformer)
646
643
647 line = self.assemble_python_lines.push(line)
644 line = self.assemble_python_lines.push(line)
648 if line is None:
645 if line is None:
649 self.within_python_line = True
646 self.within_python_line = True
650 return _accumulating('acc python line')
647 return _accumulating('acc python line')
651 else:
648 else:
652 self.within_python_line = False
649 self.within_python_line = False
653
650
654 for transformer in self.python_line_transforms:
651 for transformer in self.python_line_transforms:
655 line = transformer.push(line)
652 line = transformer.push(line)
656 if line is None:
653 if line is None:
657 return _accumulating(transformer)
654 return _accumulating(transformer)
658
655
659 #print("transformers clear") #debug
656 #print("transformers clear") #debug
660 self.transformer_accumulating = False
657 self.transformer_accumulating = False
661 return super(IPythonInputSplitter, self).push(line)
658 return super(IPythonInputSplitter, self).push(line)
General Comments 0
You need to be logged in to leave comments. Login now