##// END OF EJS Templates
Avoid trying to compile the code after each line
Thomas Kluyver -
Show More
@@ -1,744 +1,759 b''
1 """Input handling and transformation machinery.
1 """Input handling and transformation machinery.
2
2
3 The first class in this module, :class:`InputSplitter`, is designed to tell when
3 The first class in this module, :class:`InputSplitter`, is designed to tell when
4 input from a line-oriented frontend is complete and should be executed, and when
4 input from a line-oriented frontend is complete and should be executed, and when
5 the user should be prompted for another line of code instead. The name 'input
5 the user should be prompted for another line of code instead. The name 'input
6 splitter' is largely for historical reasons.
6 splitter' is largely for historical reasons.
7
7
8 A companion, :class:`IPythonInputSplitter`, provides the same functionality but
8 A companion, :class:`IPythonInputSplitter`, provides the same functionality but
9 with full support for the extended IPython syntax (magics, system calls, etc).
9 with full support for the extended IPython syntax (magics, system calls, etc).
10 The code to actually do these transformations is in :mod:`IPython.core.inputtransformer`.
10 The code to actually do these transformations is in :mod:`IPython.core.inputtransformer`.
11 :class:`IPythonInputSplitter` feeds the raw code to the transformers in order
11 :class:`IPythonInputSplitter` feeds the raw code to the transformers in order
12 and stores the results.
12 and stores the results.
13
13
14 For more details, see the class docstrings below.
14 For more details, see the class docstrings below.
15 """
15 """
16
16
17 # Copyright (c) IPython Development Team.
17 # Copyright (c) IPython Development Team.
18 # Distributed under the terms of the Modified BSD License.
18 # Distributed under the terms of the Modified BSD License.
19 import ast
19 import ast
20 import codeop
20 import codeop
21 import io
21 import io
22 import re
22 import re
23 import sys
23 import sys
24 import tokenize
24 import tokenize
25 import warnings
25 import warnings
26
26
27 from IPython.utils.py3compat import cast_unicode
27 from IPython.utils.py3compat import cast_unicode
28 from IPython.core.inputtransformer import (leading_indent,
28 from IPython.core.inputtransformer import (leading_indent,
29 classic_prompt,
29 classic_prompt,
30 ipy_prompt,
30 ipy_prompt,
31 cellmagic,
31 cellmagic,
32 assemble_logical_lines,
32 assemble_logical_lines,
33 help_end,
33 help_end,
34 escaped_commands,
34 escaped_commands,
35 assign_from_magic,
35 assign_from_magic,
36 assign_from_system,
36 assign_from_system,
37 assemble_python_lines,
37 assemble_python_lines,
38 )
38 )
39
39
40 # These are available in this module for backwards compatibility.
40 # These are available in this module for backwards compatibility.
41 from IPython.core.inputtransformer import (ESC_SHELL, ESC_SH_CAP, ESC_HELP,
41 from IPython.core.inputtransformer import (ESC_SHELL, ESC_SH_CAP, ESC_HELP,
42 ESC_HELP2, ESC_MAGIC, ESC_MAGIC2,
42 ESC_HELP2, ESC_MAGIC, ESC_MAGIC2,
43 ESC_QUOTE, ESC_QUOTE2, ESC_PAREN, ESC_SEQUENCES)
43 ESC_QUOTE, ESC_QUOTE2, ESC_PAREN, ESC_SEQUENCES)
44
44
45 #-----------------------------------------------------------------------------
45 #-----------------------------------------------------------------------------
46 # Utilities
46 # Utilities
47 #-----------------------------------------------------------------------------
47 #-----------------------------------------------------------------------------
48
48
49 # FIXME: These are general-purpose utilities that later can be moved to the
49 # FIXME: These are general-purpose utilities that later can be moved to the
50 # general ward. Kept here for now because we're being very strict about test
50 # general ward. Kept here for now because we're being very strict about test
51 # coverage with this code, and this lets us ensure that we keep 100% coverage
51 # coverage with this code, and this lets us ensure that we keep 100% coverage
52 # while developing.
52 # while developing.
53
53
54 # compiled regexps for autoindent management
54 # compiled regexps for autoindent management
55 dedent_re = re.compile('|'.join([
55 dedent_re = re.compile('|'.join([
56 r'^\s+raise(\s.*)?$', # raise statement (+ space + other stuff, maybe)
56 r'^\s+raise(\s.*)?$', # raise statement (+ space + other stuff, maybe)
57 r'^\s+raise\([^\)]*\).*$', # wacky raise with immediate open paren
57 r'^\s+raise\([^\)]*\).*$', # wacky raise with immediate open paren
58 r'^\s+return(\s.*)?$', # normal return (+ space + other stuff, maybe)
58 r'^\s+return(\s.*)?$', # normal return (+ space + other stuff, maybe)
59 r'^\s+return\([^\)]*\).*$', # wacky return with immediate open paren
59 r'^\s+return\([^\)]*\).*$', # wacky return with immediate open paren
60 r'^\s+pass\s*$', # pass (optionally followed by trailing spaces)
60 r'^\s+pass\s*$', # pass (optionally followed by trailing spaces)
61 r'^\s+break\s*$', # break (optionally followed by trailing spaces)
61 r'^\s+break\s*$', # break (optionally followed by trailing spaces)
62 r'^\s+continue\s*$', # continue (optionally followed by trailing spaces)
62 r'^\s+continue\s*$', # continue (optionally followed by trailing spaces)
63 ]))
63 ]))
64 ini_spaces_re = re.compile(r'^([ \t\r\f\v]+)')
64 ini_spaces_re = re.compile(r'^([ \t\r\f\v]+)')
65
65
66 # regexp to match pure comment lines so we don't accidentally insert 'if 1:'
66 # regexp to match pure comment lines so we don't accidentally insert 'if 1:'
67 # before pure comments
67 # before pure comments
68 comment_line_re = re.compile('^\s*\#')
68 comment_line_re = re.compile('^\s*\#')
69
69
70
70
71 def num_ini_spaces(s):
71 def num_ini_spaces(s):
72 """Return the number of initial spaces in a string.
72 """Return the number of initial spaces in a string.
73
73
74 Note that tabs are counted as a single space. For now, we do *not* support
74 Note that tabs are counted as a single space. For now, we do *not* support
75 mixing of tabs and spaces in the user's input.
75 mixing of tabs and spaces in the user's input.
76
76
77 Parameters
77 Parameters
78 ----------
78 ----------
79 s : string
79 s : string
80
80
81 Returns
81 Returns
82 -------
82 -------
83 n : int
83 n : int
84 """
84 """
85
85
86 ini_spaces = ini_spaces_re.match(s)
86 ini_spaces = ini_spaces_re.match(s)
87 if ini_spaces:
87 if ini_spaces:
88 return ini_spaces.end()
88 return ini_spaces.end()
89 else:
89 else:
90 return 0
90 return 0
91
91
92 # Fake token types for partial_tokenize:
92 # Fake token types for partial_tokenize:
93 INCOMPLETE_STRING = tokenize.N_TOKENS
93 INCOMPLETE_STRING = tokenize.N_TOKENS
94 IN_MULTILINE_STATEMENT = tokenize.N_TOKENS + 1
94 IN_MULTILINE_STATEMENT = tokenize.N_TOKENS + 1
95
95
96 # The 2 classes below have the same API as TokenInfo, but don't try to look up
96 # The 2 classes below have the same API as TokenInfo, but don't try to look up
97 # a token type name that they won't find.
97 # a token type name that they won't find.
98 class IncompleteString:
98 class IncompleteString:
99 type = exact_type = INCOMPLETE_STRING
99 type = exact_type = INCOMPLETE_STRING
100 def __init__(self, s, start, end, line):
100 def __init__(self, s, start, end, line):
101 self.s = s
101 self.s = s
102 self.start = start
102 self.start = start
103 self.end = end
103 self.end = end
104 self.line = line
104 self.line = line
105
105
106 class InMultilineStatement:
106 class InMultilineStatement:
107 type = exact_type = IN_MULTILINE_STATEMENT
107 type = exact_type = IN_MULTILINE_STATEMENT
108 def __init__(self, pos, line):
108 def __init__(self, pos, line):
109 self.s = ''
109 self.s = ''
110 self.start = self.end = pos
110 self.start = self.end = pos
111 self.line = line
111 self.line = line
112
112
113 def partial_tokens(s):
113 def partial_tokens(s):
114 """Iterate over tokens from a possibly-incomplete string of code.
114 """Iterate over tokens from a possibly-incomplete string of code.
115
115
116 This adds two special token types: INCOMPLETE_STRING and
116 This adds two special token types: INCOMPLETE_STRING and
117 IN_MULTILINE_STATEMENT. These can only occur as the last token yielded, and
117 IN_MULTILINE_STATEMENT. These can only occur as the last token yielded, and
118 represent the two main ways for code to be incomplete.
118 represent the two main ways for code to be incomplete.
119 """
119 """
120 readline = io.StringIO(s).readline
120 readline = io.StringIO(s).readline
121 token = tokenize.TokenInfo(tokenize.NEWLINE, '', (1, 0), (1, 0), '')
121 token = tokenize.TokenInfo(tokenize.NEWLINE, '', (1, 0), (1, 0), '')
122 try:
122 try:
123 for token in tokenize.generate_tokens(readline):
123 for token in tokenize.generate_tokens(readline):
124 yield token
124 yield token
125 except tokenize.TokenError as e:
125 except tokenize.TokenError as e:
126 # catch EOF error
126 # catch EOF error
127 lines = s.splitlines(keepends=True)
127 lines = s.splitlines(keepends=True)
128 end = len(lines), len(lines[-1])
128 end = len(lines), len(lines[-1])
129 if 'multi-line string' in e.args[0]:
129 if 'multi-line string' in e.args[0]:
130 l, c = start = token.end
130 l, c = start = token.end
131 s = lines[l-1][c:] + ''.join(lines[l:])
131 s = lines[l-1][c:] + ''.join(lines[l:])
132 yield IncompleteString(s, start, end, lines[-1])
132 yield IncompleteString(s, start, end, lines[-1])
133 elif 'multi-line statement' in e.args[0]:
133 elif 'multi-line statement' in e.args[0]:
134 yield InMultilineStatement(end, lines[-1])
134 yield InMultilineStatement(end, lines[-1])
135 else:
135 else:
136 raise
136 raise
137
137
138 def find_next_indent(code):
138 def find_next_indent(code):
139 """Find the number of spaces for the next line of indentation"""
139 """Find the number of spaces for the next line of indentation"""
140 tokens = list(partial_tokens(code))
140 tokens = list(partial_tokens(code))
141 if tokens[-1].type == tokenize.ENDMARKER:
141 if tokens[-1].type == tokenize.ENDMARKER:
142 tokens.pop()
142 tokens.pop()
143 if not tokens:
143 if not tokens:
144 return 0
144 return 0
145 while (tokens[-1].type in {tokenize.DEDENT, tokenize.NEWLINE, tokenize.COMMENT}):
145 while (tokens[-1].type in {tokenize.DEDENT, tokenize.NEWLINE, tokenize.COMMENT}):
146 tokens.pop()
146 tokens.pop()
147
147
148 if tokens[-1].type == INCOMPLETE_STRING:
148 if tokens[-1].type == INCOMPLETE_STRING:
149 # Inside a multiline string
149 # Inside a multiline string
150 return 0
150 return 0
151
151
152 # Find the indents used before
152 # Find the indents used before
153 prev_indents = [0]
153 prev_indents = [0]
154 def _add_indent(n):
154 def _add_indent(n):
155 if n != prev_indents[-1]:
155 if n != prev_indents[-1]:
156 prev_indents.append(n)
156 prev_indents.append(n)
157
157
158 tokiter = iter(tokens)
158 tokiter = iter(tokens)
159 for tok in tokiter:
159 for tok in tokiter:
160 if tok.type in {tokenize.INDENT, tokenize.DEDENT}:
160 if tok.type in {tokenize.INDENT, tokenize.DEDENT}:
161 _add_indent(tok.end[1])
161 _add_indent(tok.end[1])
162 elif (tok.type == tokenize.NL):
162 elif (tok.type == tokenize.NL):
163 try:
163 try:
164 _add_indent(next(tokiter).start[1])
164 _add_indent(next(tokiter).start[1])
165 except StopIteration:
165 except StopIteration:
166 break
166 break
167
167
168 last_indent = prev_indents.pop()
168 last_indent = prev_indents.pop()
169
169
170 # If we've just opened a multiline statement (e.g. 'a = ['), indent more
170 # If we've just opened a multiline statement (e.g. 'a = ['), indent more
171 if tokens[-1].type == IN_MULTILINE_STATEMENT:
171 if tokens[-1].type == IN_MULTILINE_STATEMENT:
172 if tokens[-2].exact_type in {tokenize.LPAR, tokenize.LSQB, tokenize.LBRACE}:
172 if tokens[-2].exact_type in {tokenize.LPAR, tokenize.LSQB, tokenize.LBRACE}:
173 return last_indent + 4
173 return last_indent + 4
174 return last_indent
174 return last_indent
175
175
176 if tokens[-1].exact_type == tokenize.COLON:
176 if tokens[-1].exact_type == tokenize.COLON:
177 # Line ends with colon - indent
177 # Line ends with colon - indent
178 return last_indent + 4
178 return last_indent + 4
179
179
180 if last_indent:
180 if last_indent:
181 # Examine the last line for dedent cues - statements like return or
181 # Examine the last line for dedent cues - statements like return or
182 # raise which normally end a block of code.
182 # raise which normally end a block of code.
183 last_line_starts = 0
183 last_line_starts = 0
184 for i, tok in enumerate(tokens):
184 for i, tok in enumerate(tokens):
185 if tok.type == tokenize.NEWLINE:
185 if tok.type == tokenize.NEWLINE:
186 last_line_starts = i + 1
186 last_line_starts = i + 1
187
187
188 last_line_tokens = tokens[last_line_starts:]
188 last_line_tokens = tokens[last_line_starts:]
189 names = [t.string for t in last_line_tokens if t.type == tokenize.NAME]
189 names = [t.string for t in last_line_tokens if t.type == tokenize.NAME]
190 if names and names[0] in {'raise', 'return', 'pass', 'break', 'continue'}:
190 if names and names[0] in {'raise', 'return', 'pass', 'break', 'continue'}:
191 # Find the most recent indentation less than the current level
191 # Find the most recent indentation less than the current level
192 for indent in reversed(prev_indents):
192 for indent in reversed(prev_indents):
193 if indent < last_indent:
193 if indent < last_indent:
194 return indent
194 return indent
195
195
196 return last_indent
196 return last_indent
197
197
198
198
199 def last_blank(src):
199 def last_blank(src):
200 """Determine if the input source ends in a blank.
200 """Determine if the input source ends in a blank.
201
201
202 A blank is either a newline or a line consisting of whitespace.
202 A blank is either a newline or a line consisting of whitespace.
203
203
204 Parameters
204 Parameters
205 ----------
205 ----------
206 src : string
206 src : string
207 A single or multiline string.
207 A single or multiline string.
208 """
208 """
209 if not src: return False
209 if not src: return False
210 ll = src.splitlines()[-1]
210 ll = src.splitlines()[-1]
211 return (ll == '') or ll.isspace()
211 return (ll == '') or ll.isspace()
212
212
213
213
214 last_two_blanks_re = re.compile(r'\n\s*\n\s*$', re.MULTILINE)
214 last_two_blanks_re = re.compile(r'\n\s*\n\s*$', re.MULTILINE)
215 last_two_blanks_re2 = re.compile(r'.+\n\s*\n\s+$', re.MULTILINE)
215 last_two_blanks_re2 = re.compile(r'.+\n\s*\n\s+$', re.MULTILINE)
216
216
217 def last_two_blanks(src):
217 def last_two_blanks(src):
218 """Determine if the input source ends in two blanks.
218 """Determine if the input source ends in two blanks.
219
219
220 A blank is either a newline or a line consisting of whitespace.
220 A blank is either a newline or a line consisting of whitespace.
221
221
222 Parameters
222 Parameters
223 ----------
223 ----------
224 src : string
224 src : string
225 A single or multiline string.
225 A single or multiline string.
226 """
226 """
227 if not src: return False
227 if not src: return False
228 # The logic here is tricky: I couldn't get a regexp to work and pass all
228 # The logic here is tricky: I couldn't get a regexp to work and pass all
229 # the tests, so I took a different approach: split the source by lines,
229 # the tests, so I took a different approach: split the source by lines,
230 # grab the last two and prepend '###\n' as a stand-in for whatever was in
230 # grab the last two and prepend '###\n' as a stand-in for whatever was in
231 # the body before the last two lines. Then, with that structure, it's
231 # the body before the last two lines. Then, with that structure, it's
232 # possible to analyze with two regexps. Not the most elegant solution, but
232 # possible to analyze with two regexps. Not the most elegant solution, but
233 # it works. If anyone tries to change this logic, make sure to validate
233 # it works. If anyone tries to change this logic, make sure to validate
234 # the whole test suite first!
234 # the whole test suite first!
235 new_src = '\n'.join(['###\n'] + src.splitlines()[-2:])
235 new_src = '\n'.join(['###\n'] + src.splitlines()[-2:])
236 return (bool(last_two_blanks_re.match(new_src)) or
236 return (bool(last_two_blanks_re.match(new_src)) or
237 bool(last_two_blanks_re2.match(new_src)) )
237 bool(last_two_blanks_re2.match(new_src)) )
238
238
239
239
240 def remove_comments(src):
240 def remove_comments(src):
241 """Remove all comments from input source.
241 """Remove all comments from input source.
242
242
243 Note: comments are NOT recognized inside of strings!
243 Note: comments are NOT recognized inside of strings!
244
244
245 Parameters
245 Parameters
246 ----------
246 ----------
247 src : string
247 src : string
248 A single or multiline input string.
248 A single or multiline input string.
249
249
250 Returns
250 Returns
251 -------
251 -------
252 String with all Python comments removed.
252 String with all Python comments removed.
253 """
253 """
254
254
255 return re.sub('#.*', '', src)
255 return re.sub('#.*', '', src)
256
256
257
257
258 def get_input_encoding():
258 def get_input_encoding():
259 """Return the default standard input encoding.
259 """Return the default standard input encoding.
260
260
261 If sys.stdin has no encoding, 'ascii' is returned."""
261 If sys.stdin has no encoding, 'ascii' is returned."""
262 # There are strange environments for which sys.stdin.encoding is None. We
262 # There are strange environments for which sys.stdin.encoding is None. We
263 # ensure that a valid encoding is returned.
263 # ensure that a valid encoding is returned.
264 encoding = getattr(sys.stdin, 'encoding', None)
264 encoding = getattr(sys.stdin, 'encoding', None)
265 if encoding is None:
265 if encoding is None:
266 encoding = 'ascii'
266 encoding = 'ascii'
267 return encoding
267 return encoding
268
268
269 #-----------------------------------------------------------------------------
269 #-----------------------------------------------------------------------------
270 # Classes and functions for normal Python syntax handling
270 # Classes and functions for normal Python syntax handling
271 #-----------------------------------------------------------------------------
271 #-----------------------------------------------------------------------------
272
272
273 class InputSplitter(object):
273 class InputSplitter(object):
274 r"""An object that can accumulate lines of Python source before execution.
274 r"""An object that can accumulate lines of Python source before execution.
275
275
276 This object is designed to be fed python source line-by-line, using
276 This object is designed to be fed python source line-by-line, using
277 :meth:`push`. It will return on each push whether the currently pushed
277 :meth:`push`. It will return on each push whether the currently pushed
278 code could be executed already. In addition, it provides a method called
278 code could be executed already. In addition, it provides a method called
279 :meth:`push_accepts_more` that can be used to query whether more input
279 :meth:`push_accepts_more` that can be used to query whether more input
280 can be pushed into a single interactive block.
280 can be pushed into a single interactive block.
281
281
282 This is a simple example of how an interactive terminal-based client can use
282 This is a simple example of how an interactive terminal-based client can use
283 this tool::
283 this tool::
284
284
285 isp = InputSplitter()
285 isp = InputSplitter()
286 while isp.push_accepts_more():
286 while isp.push_accepts_more():
287 indent = ' '*isp.indent_spaces
287 indent = ' '*isp.indent_spaces
288 prompt = '>>> ' + indent
288 prompt = '>>> ' + indent
289 line = indent + raw_input(prompt)
289 line = indent + raw_input(prompt)
290 isp.push(line)
290 isp.push(line)
291 print 'Input source was:\n', isp.source_reset(),
291 print 'Input source was:\n', isp.source_reset(),
292 """
292 """
293 # A cache for calculating the current indentation.
293 # A cache for calculating the current indentation.
294 # If the first value matches self.source, the second value is an integer
294 # If the first value matches self.source, the second value is an integer
295 # number of spaces for the current indentation. If the first value does not
295 # number of spaces for the current indentation. If the first value does not
296 # match, self.source has changed, and the indentation must be recalculated.
296 # match, self.source has changed, and the indentation must be recalculated.
297 _indent_spaces_cache = None, None
297 _indent_spaces_cache = None, None
298 # String, indicating the default input encoding. It is computed by default
298 # String, indicating the default input encoding. It is computed by default
299 # at initialization time via get_input_encoding(), but it can be reset by a
299 # at initialization time via get_input_encoding(), but it can be reset by a
300 # client with specific knowledge of the encoding.
300 # client with specific knowledge of the encoding.
301 encoding = ''
301 encoding = ''
302 # String where the current full source input is stored, properly encoded.
302 # String where the current full source input is stored, properly encoded.
303 # Reading this attribute is the normal way of querying the currently pushed
303 # Reading this attribute is the normal way of querying the currently pushed
304 # source code, that has been properly encoded.
304 # source code, that has been properly encoded.
305 source = ''
305 source = ''
306 # Code object corresponding to the current source. It is automatically
306 # Code object corresponding to the current source. It is automatically
307 # synced to the source, so it can be queried at any time to obtain the code
307 # synced to the source, so it can be queried at any time to obtain the code
308 # object; it will be None if the source doesn't compile to valid Python.
308 # object; it will be None if the source doesn't compile to valid Python.
309 code = None
309 code = None
310
310
311 # Private attributes
311 # Private attributes
312
312
313 # List with lines of input accumulated so far
313 # List with lines of input accumulated so far
314 _buffer = None
314 _buffer = None
315 # Command compiler
315 # Command compiler
316 _compile = None
316 _compile = None
317 # Boolean indicating whether the current block is complete
317 # Boolean indicating whether the current block is complete
318 _is_complete = None
318 _is_complete = None
319 # Boolean indicating whether the current block has an unrecoverable syntax error
319 # Boolean indicating whether the current block has an unrecoverable syntax error
320 _is_invalid = False
320 _is_invalid = False
321
321
322 def __init__(self):
322 def __init__(self):
323 """Create a new InputSplitter instance.
323 """Create a new InputSplitter instance.
324 """
324 """
325 self._buffer = []
325 self._buffer = []
326 self._compile = codeop.CommandCompiler()
326 self._compile = codeop.CommandCompiler()
327 self.encoding = get_input_encoding()
327 self.encoding = get_input_encoding()
328
328
329 def reset(self):
329 def reset(self):
330 """Reset the input buffer and associated state."""
330 """Reset the input buffer and associated state."""
331 self._buffer[:] = []
331 self._buffer[:] = []
332 self.source = ''
332 self.source = ''
333 self.code = None
333 self.code = None
334 self._is_complete = False
334 self._is_complete = False
335 self._is_invalid = False
335 self._is_invalid = False
336
336
337 def source_reset(self):
337 def source_reset(self):
338 """Return the input source and perform a full reset.
338 """Return the input source and perform a full reset.
339 """
339 """
340 out = self.source
340 out = self.source
341 self.reset()
341 self.reset()
342 return out
342 return out
343
343
344 def check_complete(self, source):
344 def check_complete(self, source):
345 """Return whether a block of code is ready to execute, or should be continued
345 """Return whether a block of code is ready to execute, or should be continued
346
346
347 This is a non-stateful API, and will reset the state of this InputSplitter.
347 This is a non-stateful API, and will reset the state of this InputSplitter.
348
348
349 Parameters
349 Parameters
350 ----------
350 ----------
351 source : string
351 source : string
352 Python input code, which can be multiline.
352 Python input code, which can be multiline.
353
353
354 Returns
354 Returns
355 -------
355 -------
356 status : str
356 status : str
357 One of 'complete', 'incomplete', or 'invalid' if source is not a
357 One of 'complete', 'incomplete', or 'invalid' if source is not a
358 prefix of valid code.
358 prefix of valid code.
359 indent_spaces : int or None
359 indent_spaces : int or None
360 The number of spaces by which to indent the next line of code. If
360 The number of spaces by which to indent the next line of code. If
361 status is not 'incomplete', this is None.
361 status is not 'incomplete', this is None.
362 """
362 """
363 self.reset()
363 self.reset()
364 try:
364 try:
365 self.push(source)
365 self.push(source)
366 except SyntaxError:
366 except SyntaxError:
367 # Transformers in IPythonInputSplitter can raise SyntaxError,
367 # Transformers in IPythonInputSplitter can raise SyntaxError,
368 # which push() will not catch.
368 # which push() will not catch.
369 return 'invalid', None
369 return 'invalid', None
370 else:
370 else:
371 if self._is_invalid:
371 if self._is_invalid:
372 return 'invalid', None
372 return 'invalid', None
373 elif self.push_accepts_more():
373 elif self.push_accepts_more():
374 return 'incomplete', self.get_indent_spaces()
374 return 'incomplete', self.get_indent_spaces()
375 else:
375 else:
376 return 'complete', None
376 return 'complete', None
377 finally:
377 finally:
378 self.reset()
378 self.reset()
379
379
380 def push(self, lines):
380 def push(self, lines):
381 """Push one or more lines of input.
381 """Push one or more lines of input.
382
382
383 This stores the given lines and returns a status code indicating
383 This stores the given lines and returns a status code indicating
384 whether the code forms a complete Python block or not.
384 whether the code forms a complete Python block or not.
385
385
386 Any exceptions generated in compilation are swallowed, but if an
386 Any exceptions generated in compilation are swallowed, but if an
387 exception was produced, the method returns True.
387 exception was produced, the method returns True.
388
388
389 Parameters
389 Parameters
390 ----------
390 ----------
391 lines : string
391 lines : string
392 One or more lines of Python input.
392 One or more lines of Python input.
393
393
394 Returns
394 Returns
395 -------
395 -------
396 is_complete : boolean
396 is_complete : boolean
397 True if the current input source (the result of the current input
397 True if the current input source (the result of the current input
398 plus prior inputs) forms a complete Python execution block. Note that
398 plus prior inputs) forms a complete Python execution block. Note that
399 this value is also stored as a private attribute (``_is_complete``), so it
399 this value is also stored as a private attribute (``_is_complete``), so it
400 can be queried at any time.
400 can be queried at any time.
401 """
401 """
402 self._store(lines)
402 self._store(lines)
403 source = self.source
403 source = self.source
404
404
405 # Before calling _compile(), reset the code object to None so that if an
405 # Before calling _compile(), reset the code object to None so that if an
406 # exception is raised in compilation, we don't mislead by having
406 # exception is raised in compilation, we don't mislead by having
407 # inconsistent code/source attributes.
407 # inconsistent code/source attributes.
408 self.code, self._is_complete = None, None
408 self.code, self._is_complete = None, None
409 self._is_invalid = False
409 self._is_invalid = False
410
410
411 # Honor termination lines properly
411 # Honor termination lines properly
412 if source.endswith('\\\n'):
412 if source.endswith('\\\n'):
413 return False
413 return False
414
414
415 try:
415 try:
416 with warnings.catch_warnings():
416 with warnings.catch_warnings():
417 warnings.simplefilter('error', SyntaxWarning)
417 warnings.simplefilter('error', SyntaxWarning)
418 self.code = self._compile(source, symbol="exec")
418 self.code = self._compile(source, symbol="exec")
419 # Invalid syntax can produce any of a number of different errors from
419 # Invalid syntax can produce any of a number of different errors from
420 # inside the compiler, so we have to catch them all. Syntax errors
420 # inside the compiler, so we have to catch them all. Syntax errors
421 # immediately produce a 'ready' block, so the invalid Python can be
421 # immediately produce a 'ready' block, so the invalid Python can be
422 # sent to the kernel for evaluation with possible ipython
422 # sent to the kernel for evaluation with possible ipython
423 # special-syntax conversion.
423 # special-syntax conversion.
424 except (SyntaxError, OverflowError, ValueError, TypeError,
424 except (SyntaxError, OverflowError, ValueError, TypeError,
425 MemoryError, SyntaxWarning):
425 MemoryError, SyntaxWarning):
426 self._is_complete = True
426 self._is_complete = True
427 self._is_invalid = True
427 self._is_invalid = True
428 else:
428 else:
429 # Compilation didn't produce any exceptions (though it may not have
429 # Compilation didn't produce any exceptions (though it may not have
430 # given a complete code object)
430 # given a complete code object)
431 self._is_complete = self.code is not None
431 self._is_complete = self.code is not None
432
432
433 return self._is_complete
433 return self._is_complete
434
434
435 def push_accepts_more(self):
435 def push_accepts_more(self):
436 """Return whether a block of interactive input can accept more input.
436 """Return whether a block of interactive input can accept more input.
437
437
438 This method is meant to be used by line-oriented frontends, who need to
438 This method is meant to be used by line-oriented frontends, who need to
439 guess whether a block is complete or not based solely on prior and
439 guess whether a block is complete or not based solely on prior and
440 current input lines. The InputSplitter considers it has a complete
440 current input lines. The InputSplitter considers it has a complete
441 interactive block and will not accept more input when either:
441 interactive block and will not accept more input when either:
442
442
443 * A SyntaxError is raised
443 * A SyntaxError is raised
444
444
445 * The code is complete and consists of a single line or a single
445 * The code is complete and consists of a single line or a single
446 non-compound statement
446 non-compound statement
447
447
448 * The code is complete and has a blank line at the end
448 * The code is complete and has a blank line at the end
449
449
450 If the current input produces a syntax error, this method immediately
450 If the current input produces a syntax error, this method immediately
451 returns False but does *not* raise the syntax error exception, as
451 returns False but does *not* raise the syntax error exception, as
452 typically clients will want to send invalid syntax to an execution
452 typically clients will want to send invalid syntax to an execution
453 backend which might convert the invalid syntax into valid Python via
453 backend which might convert the invalid syntax into valid Python via
454 one of the dynamic IPython mechanisms.
454 one of the dynamic IPython mechanisms.
455 """
455 """
456
456
457 # With incomplete input, unconditionally accept more
457 # With incomplete input, unconditionally accept more
458 # A syntax error also sets _is_complete to True - see push()
458 # A syntax error also sets _is_complete to True - see push()
459 if not self._is_complete:
459 if not self._is_complete:
460 #print("Not complete") # debug
460 #print("Not complete") # debug
461 return True
461 return True
462
462
463 # The user can make any (complete) input execute by leaving a blank line
463 # The user can make any (complete) input execute by leaving a blank line
464 last_line = self.source.splitlines()[-1]
464 last_line = self.source.splitlines()[-1]
465 if (not last_line) or last_line.isspace():
465 if (not last_line) or last_line.isspace():
466 #print("Blank line") # debug
466 #print("Blank line") # debug
467 return False
467 return False
468
468
469 # If there's just a single line or AST node, and we're flush left, as is
469 # If there's just a single line or AST node, and we're flush left, as is
470 # the case after a simple statement such as 'a=1', we want to execute it
470 # the case after a simple statement such as 'a=1', we want to execute it
471 # straight away.
471 # straight away.
472 if self.get_indent_spaces() == 0:
472 if self.get_indent_spaces() == 0:
473 if len(self.source.splitlines()) <= 1:
473 if len(self.source.splitlines()) <= 1:
474 return False
474 return False
475
475
476 try:
476 try:
477 code_ast = ast.parse(u''.join(self._buffer))
477 code_ast = ast.parse(u''.join(self._buffer))
478 except Exception:
478 except Exception:
479 #print("Can't parse AST") # debug
479 #print("Can't parse AST") # debug
480 return False
480 return False
481 else:
481 else:
482 if len(code_ast.body) == 1 and \
482 if len(code_ast.body) == 1 and \
483 not hasattr(code_ast.body[0], 'body'):
483 not hasattr(code_ast.body[0], 'body'):
484 #print("Simple statement") # debug
484 #print("Simple statement") # debug
485 return False
485 return False
486
486
487 # General fallback - accept more code
487 # General fallback - accept more code
488 return True
488 return True
489
489
490 def get_indent_spaces(self):
490 def get_indent_spaces(self):
491 sourcefor, n = self._indent_spaces_cache
491 sourcefor, n = self._indent_spaces_cache
492 if sourcefor == self.source:
492 if sourcefor == self.source:
493 return n
493 return n
494
494
495 # self.source always has a trailing newline
495 # self.source always has a trailing newline
496 n = find_next_indent(self.source[:-1])
496 n = find_next_indent(self.source[:-1])
497 self._indent_spaces_cache = (self.source, n)
497 self._indent_spaces_cache = (self.source, n)
498 return n
498 return n
499
499
500 def _store(self, lines, buffer=None, store='source'):
500 def _store(self, lines, buffer=None, store='source'):
501 """Store one or more lines of input.
501 """Store one or more lines of input.
502
502
503 If input lines are not newline-terminated, a newline is automatically
503 If input lines are not newline-terminated, a newline is automatically
504 appended."""
504 appended."""
505
505
506 if buffer is None:
506 if buffer is None:
507 buffer = self._buffer
507 buffer = self._buffer
508
508
509 if lines.endswith('\n'):
509 if lines.endswith('\n'):
510 buffer.append(lines)
510 buffer.append(lines)
511 else:
511 else:
512 buffer.append(lines+'\n')
512 buffer.append(lines+'\n')
513 setattr(self, store, self._set_source(buffer))
513 setattr(self, store, self._set_source(buffer))
514
514
515 def _set_source(self, buffer):
515 def _set_source(self, buffer):
516 return u''.join(buffer)
516 return u''.join(buffer)
517
517
518
518
519 class IPythonInputSplitter(InputSplitter):
519 class IPythonInputSplitter(InputSplitter):
520 """An input splitter that recognizes all of IPython's special syntax."""
520 """An input splitter that recognizes all of IPython's special syntax."""
521
521
522 # String with raw, untransformed input.
522 # String with raw, untransformed input.
523 source_raw = ''
523 source_raw = ''
524
524
525 # Flag to track when a transformer has stored input that it hasn't given
525 # Flag to track when a transformer has stored input that it hasn't given
526 # back yet.
526 # back yet.
527 transformer_accumulating = False
527 transformer_accumulating = False
528
528
529 # Flag to track when assemble_python_lines has stored input that it hasn't
529 # Flag to track when assemble_python_lines has stored input that it hasn't
530 # given back yet.
530 # given back yet.
531 within_python_line = False
531 within_python_line = False
532
532
533 # Private attributes
533 # Private attributes
534
534
535 # List with lines of raw input accumulated so far.
535 # List with lines of raw input accumulated so far.
536 _buffer_raw = None
536 _buffer_raw = None
537
537
538 def __init__(self, line_input_checker=True, physical_line_transforms=None,
538 def __init__(self, line_input_checker=True, physical_line_transforms=None,
539 logical_line_transforms=None, python_line_transforms=None):
539 logical_line_transforms=None, python_line_transforms=None):
540 super(IPythonInputSplitter, self).__init__()
540 super(IPythonInputSplitter, self).__init__()
541 self._buffer_raw = []
541 self._buffer_raw = []
542 self._validate = True
542 self._validate = True
543
543
544 if physical_line_transforms is not None:
544 if physical_line_transforms is not None:
545 self.physical_line_transforms = physical_line_transforms
545 self.physical_line_transforms = physical_line_transforms
546 else:
546 else:
547 self.physical_line_transforms = [
547 self.physical_line_transforms = [
548 leading_indent(),
548 leading_indent(),
549 classic_prompt(),
549 classic_prompt(),
550 ipy_prompt(),
550 ipy_prompt(),
551 cellmagic(end_on_blank_line=line_input_checker),
551 cellmagic(end_on_blank_line=line_input_checker),
552 ]
552 ]
553
553
554 self.assemble_logical_lines = assemble_logical_lines()
554 self.assemble_logical_lines = assemble_logical_lines()
555 if logical_line_transforms is not None:
555 if logical_line_transforms is not None:
556 self.logical_line_transforms = logical_line_transforms
556 self.logical_line_transforms = logical_line_transforms
557 else:
557 else:
558 self.logical_line_transforms = [
558 self.logical_line_transforms = [
559 help_end(),
559 help_end(),
560 escaped_commands(),
560 escaped_commands(),
561 assign_from_magic(),
561 assign_from_magic(),
562 assign_from_system(),
562 assign_from_system(),
563 ]
563 ]
564
564
565 self.assemble_python_lines = assemble_python_lines()
565 self.assemble_python_lines = assemble_python_lines()
566 if python_line_transforms is not None:
566 if python_line_transforms is not None:
567 self.python_line_transforms = python_line_transforms
567 self.python_line_transforms = python_line_transforms
568 else:
568 else:
569 # We don't use any of these at present
569 # We don't use any of these at present
570 self.python_line_transforms = []
570 self.python_line_transforms = []
571
571
572 @property
572 @property
573 def transforms(self):
573 def transforms(self):
574 "Quick access to all transformers."
574 "Quick access to all transformers."
575 return self.physical_line_transforms + \
575 return self.physical_line_transforms + \
576 [self.assemble_logical_lines] + self.logical_line_transforms + \
576 [self.assemble_logical_lines] + self.logical_line_transforms + \
577 [self.assemble_python_lines] + self.python_line_transforms
577 [self.assemble_python_lines] + self.python_line_transforms
578
578
579 @property
579 @property
580 def transforms_in_use(self):
580 def transforms_in_use(self):
581 """Transformers, excluding logical line transformers if we're in a
581 """Transformers, excluding logical line transformers if we're in a
582 Python line."""
582 Python line."""
583 t = self.physical_line_transforms[:]
583 t = self.physical_line_transforms[:]
584 if not self.within_python_line:
584 if not self.within_python_line:
585 t += [self.assemble_logical_lines] + self.logical_line_transforms
585 t += [self.assemble_logical_lines] + self.logical_line_transforms
586 return t + [self.assemble_python_lines] + self.python_line_transforms
586 return t + [self.assemble_python_lines] + self.python_line_transforms
587
587
588 def reset(self):
588 def reset(self):
589 """Reset the input buffer and associated state."""
589 """Reset the input buffer and associated state."""
590 super(IPythonInputSplitter, self).reset()
590 super(IPythonInputSplitter, self).reset()
591 self._buffer_raw[:] = []
591 self._buffer_raw[:] = []
592 self.source_raw = ''
592 self.source_raw = ''
593 self.transformer_accumulating = False
593 self.transformer_accumulating = False
594 self.within_python_line = False
594 self.within_python_line = False
595
595
596 for t in self.transforms:
596 for t in self.transforms:
597 try:
597 try:
598 t.reset()
598 t.reset()
599 except SyntaxError:
599 except SyntaxError:
600 # Nothing that calls reset() expects to handle transformer
600 # Nothing that calls reset() expects to handle transformer
601 # errors
601 # errors
602 pass
602 pass
603
603
604 def flush_transformers(self):
604 def flush_transformers(self):
605 def _flush(transform, outs):
605 def _flush(transform, outs):
606 """yield transformed lines
606 """yield transformed lines
607
607
608 always strings, never None
608 always strings, never None
609
609
610 transform: the current transform
610 transform: the current transform
611 outs: an iterable of previously transformed inputs.
611 outs: an iterable of previously transformed inputs.
612 Each may be multiline, which will be passed
612 Each may be multiline, which will be passed
613 one line at a time to transform.
613 one line at a time to transform.
614 """
614 """
615 for out in outs:
615 for out in outs:
616 for line in out.splitlines():
616 for line in out.splitlines():
617 # push one line at a time
617 # push one line at a time
618 tmp = transform.push(line)
618 tmp = transform.push(line)
619 if tmp is not None:
619 if tmp is not None:
620 yield tmp
620 yield tmp
621
621
622 # reset the transform
622 # reset the transform
623 tmp = transform.reset()
623 tmp = transform.reset()
624 if tmp is not None:
624 if tmp is not None:
625 yield tmp
625 yield tmp
626
626
627 out = []
627 out = []
628 for t in self.transforms_in_use:
628 for t in self.transforms_in_use:
629 out = _flush(t, out)
629 out = _flush(t, out)
630
630
631 out = list(out)
631 out = list(out)
632 if out:
632 if out:
633 self._store('\n'.join(out))
633 self._store('\n'.join(out))
634
634
635 def raw_reset(self):
635 def raw_reset(self):
636 """Return raw input only and perform a full reset.
636 """Return raw input only and perform a full reset.
637 """
637 """
638 out = self.source_raw
638 out = self.source_raw
639 self.reset()
639 self.reset()
640 return out
640 return out
641
641
642 def source_reset(self):
642 def source_reset(self):
643 try:
643 try:
644 self.flush_transformers()
644 self.flush_transformers()
645 return self.source
645 return self.source
646 finally:
646 finally:
647 self.reset()
647 self.reset()
648
648
649 def push_accepts_more(self):
649 def push_accepts_more(self):
650 if self.transformer_accumulating:
650 if self.transformer_accumulating:
651 return True
651 return True
652 else:
652 else:
653 return super(IPythonInputSplitter, self).push_accepts_more()
653 return super(IPythonInputSplitter, self).push_accepts_more()
654
654
655 def transform_cell(self, cell):
655 def transform_cell(self, cell):
656 """Process and translate a cell of input.
656 """Process and translate a cell of input.
657 """
657 """
658 self.reset()
658 self.reset()
659 try:
659 try:
660 self.push(cell)
660 self.push(cell)
661 self.flush_transformers()
661 self.flush_transformers()
662 return self.source
662 return self.source
663 finally:
663 finally:
664 self.reset()
664 self.reset()
665
665
666 def push(self, lines):
666 def push(self, lines):
667 """Push one or more lines of IPython input.
667 """Push one or more lines of IPython input.
668
668
669 This stores the given lines and returns a status code indicating
669 This stores the given lines and returns a status code indicating
670 whether the code forms a complete Python block or not, after processing
670 whether the code forms a complete Python block or not, after processing
671 all input lines for special IPython syntax.
671 all input lines for special IPython syntax.
672
672
673 Any exceptions generated in compilation are swallowed, but if an
673 Any exceptions generated in compilation are swallowed, but if an
674 exception was produced, the method returns True.
674 exception was produced, the method returns True.
675
675
676 Parameters
676 Parameters
677 ----------
677 ----------
678 lines : string
678 lines : string
679 One or more lines of Python input.
679 One or more lines of Python input.
680
680
681 Returns
681 Returns
682 -------
682 -------
683 is_complete : boolean
683 is_complete : boolean
684 True if the current input source (the result of the current input
684 True if the current input source (the result of the current input
685 plus prior inputs) forms a complete Python execution block. Note that
685 plus prior inputs) forms a complete Python execution block. Note that
686 this value is also stored as a private attribute (_is_complete), so it
686 this value is also stored as a private attribute (_is_complete), so it
687 can be queried at any time.
687 can be queried at any time.
688 """
688 """
689
689
690 # We must ensure all input is pure unicode
690 # We must ensure all input is pure unicode
691 lines = cast_unicode(lines, self.encoding)
691 lines = cast_unicode(lines, self.encoding)
692 # ''.splitlines() --> [], but we need to push the empty line to transformers
692 # ''.splitlines() --> [], but we need to push the empty line to transformers
693 lines_list = lines.splitlines()
693 lines_list = lines.splitlines()
694 if not lines_list:
694 if not lines_list:
695 lines_list = ['']
695 lines_list = ['']
696
696
697 # Store raw source before applying any transformations to it. Note
697 # Store raw source before applying any transformations to it. Note
698 # that this must be done *after* the reset() call that would otherwise
698 # that this must be done *after* the reset() call that would otherwise
699 # flush the buffer.
699 # flush the buffer.
700 self._store(lines, self._buffer_raw, 'source_raw')
700 self._store(lines, self._buffer_raw, 'source_raw')
701
701
702 transformed_lines_list = []
702 for line in lines_list:
703 for line in lines_list:
703 out = self.push_line(line)
704 transformed = self._transform_line(line)
705 if transformed is not None:
706 transformed_lines_list.append(transformed)
704
707
705 return out
708 if transformed_lines_list:
709 transformed_lines = '\n'.join(transformed_lines_list)
710 return super(IPythonInputSplitter, self).push(transformed_lines)
711 else:
712 # Got nothing back from transformers - they must be waiting for
713 # more input.
714 return False
715
716 def _transform_line(self, line):
717 """Push a line of input code through the various transformers.
706
718
707 def push_line(self, line):
719 Returns any output from the transformers, or None if a transformer
708 buf = self._buffer
720 is accumulating lines.
709
721
722 Sets self.transformer_accumulating as a side effect.
723 """
710 def _accumulating(dbg):
724 def _accumulating(dbg):
711 #print(dbg)
725 #print(dbg)
712 self.transformer_accumulating = True
726 self.transformer_accumulating = True
713 return False
727 return None
714
728
715 for transformer in self.physical_line_transforms:
729 for transformer in self.physical_line_transforms:
716 line = transformer.push(line)
730 line = transformer.push(line)
717 if line is None:
731 if line is None:
718 return _accumulating(transformer)
732 return _accumulating(transformer)
719
733
720 if not self.within_python_line:
734 if not self.within_python_line:
721 line = self.assemble_logical_lines.push(line)
735 line = self.assemble_logical_lines.push(line)
722 if line is None:
736 if line is None:
723 return _accumulating('acc logical line')
737 return _accumulating('acc logical line')
724
738
725 for transformer in self.logical_line_transforms:
739 for transformer in self.logical_line_transforms:
726 line = transformer.push(line)
740 line = transformer.push(line)
727 if line is None:
741 if line is None:
728 return _accumulating(transformer)
742 return _accumulating(transformer)
729
743
730 line = self.assemble_python_lines.push(line)
744 line = self.assemble_python_lines.push(line)
731 if line is None:
745 if line is None:
732 self.within_python_line = True
746 self.within_python_line = True
733 return _accumulating('acc python line')
747 return _accumulating('acc python line')
734 else:
748 else:
735 self.within_python_line = False
749 self.within_python_line = False
736
750
737 for transformer in self.python_line_transforms:
751 for transformer in self.python_line_transforms:
738 line = transformer.push(line)
752 line = transformer.push(line)
739 if line is None:
753 if line is None:
740 return _accumulating(transformer)
754 return _accumulating(transformer)
741
755
742 #print("transformers clear") #debug
756 #print("transformers clear") #debug
743 self.transformer_accumulating = False
757 self.transformer_accumulating = False
744 return super(IPythonInputSplitter, self).push(line)
758 return line
759
General Comments 0
You need to be logged in to leave comments. Login now