##// END OF EJS Templates
Inputsplitter flushes transformers before retrieving source.
Thomas Kluyver -
Show More
@@ -1,633 +1,649 b''
1 """Analysis of text input into executable blocks.
1 """Analysis of text input into executable blocks.
2
2
3 The main class in this module, :class:`InputSplitter`, is designed to break
3 The main class in this module, :class:`InputSplitter`, is designed to break
4 input from either interactive, line-by-line environments or block-based ones,
4 input from either interactive, line-by-line environments or block-based ones,
5 into standalone blocks that can be executed by Python as 'single' statements
5 into standalone blocks that can be executed by Python as 'single' statements
6 (thus triggering sys.displayhook).
6 (thus triggering sys.displayhook).
7
7
8 A companion, :class:`IPythonInputSplitter`, provides the same functionality but
8 A companion, :class:`IPythonInputSplitter`, provides the same functionality but
9 with full support for the extended IPython syntax (magics, system calls, etc).
9 with full support for the extended IPython syntax (magics, system calls, etc).
10
10
11 For more details, see the class docstring below.
11 For more details, see the class docstring below.
12
12
13 Syntax Transformations
13 Syntax Transformations
14 ----------------------
14 ----------------------
15
15
16 One of the main jobs of the code in this file is to apply all syntax
16 One of the main jobs of the code in this file is to apply all syntax
17 transformations that make up 'the IPython language', i.e. magics, shell
17 transformations that make up 'the IPython language', i.e. magics, shell
18 escapes, etc. All transformations should be implemented as *fully stateless*
18 escapes, etc. All transformations should be implemented as *fully stateless*
19 entities, that simply take one line as their input and return a line.
19 entities, that simply take one line as their input and return a line.
20 Internally for implementation purposes they may be a normal function or a
20 Internally for implementation purposes they may be a normal function or a
21 callable object, but the only input they receive will be a single line and they
21 callable object, but the only input they receive will be a single line and they
22 should only return a line, without holding any data-dependent state between
22 should only return a line, without holding any data-dependent state between
23 calls.
23 calls.
24
24
25 As an example, the EscapedTransformer is a class so we can more clearly group
25 As an example, the EscapedTransformer is a class so we can more clearly group
26 together the functionality of dispatching to individual functions based on the
26 together the functionality of dispatching to individual functions based on the
27 starting escape character, but the only method for public use is its call
27 starting escape character, but the only method for public use is its call
28 method.
28 method.
29
29
30
30
31 ToDo
31 ToDo
32 ----
32 ----
33
33
34 - Should we make push() actually raise an exception once push_accepts_more()
34 - Should we make push() actually raise an exception once push_accepts_more()
35 returns False?
35 returns False?
36
36
37 - Naming cleanups. The tr_* names aren't the most elegant, though now they are
37 - Naming cleanups. The tr_* names aren't the most elegant, though now they are
38 at least just attributes of a class so not really very exposed.
38 at least just attributes of a class so not really very exposed.
39
39
40 - Think about the best way to support dynamic things: automagic, autocall,
40 - Think about the best way to support dynamic things: automagic, autocall,
41 macros, etc.
41 macros, etc.
42
42
43 - Think of a better heuristic for the application of the transforms in
43 - Think of a better heuristic for the application of the transforms in
44 IPythonInputSplitter.push() than looking at the buffer ending in ':'. Idea:
44 IPythonInputSplitter.push() than looking at the buffer ending in ':'. Idea:
45 track indentation change events (indent, dedent, nothing) and apply them only
45 track indentation change events (indent, dedent, nothing) and apply them only
46 if the indentation went up, but not otherwise.
46 if the indentation went up, but not otherwise.
47
47
48 - Think of the cleanest way for supporting user-specified transformations (the
48 - Think of the cleanest way for supporting user-specified transformations (the
49 user prefilters we had before).
49 user prefilters we had before).
50
50
51 Authors
51 Authors
52 -------
52 -------
53
53
54 * Fernando Perez
54 * Fernando Perez
55 * Brian Granger
55 * Brian Granger
56 """
56 """
57 #-----------------------------------------------------------------------------
57 #-----------------------------------------------------------------------------
58 # Copyright (C) 2010 The IPython Development Team
58 # Copyright (C) 2010 The IPython Development Team
59 #
59 #
60 # Distributed under the terms of the BSD License. The full license is in
60 # Distributed under the terms of the BSD License. The full license is in
61 # the file COPYING, distributed as part of this software.
61 # the file COPYING, distributed as part of this software.
62 #-----------------------------------------------------------------------------
62 #-----------------------------------------------------------------------------
63
63
64 #-----------------------------------------------------------------------------
64 #-----------------------------------------------------------------------------
65 # Imports
65 # Imports
66 #-----------------------------------------------------------------------------
66 #-----------------------------------------------------------------------------
67 # stdlib
67 # stdlib
68 import ast
68 import ast
69 import codeop
69 import codeop
70 import re
70 import re
71 import sys
71 import sys
72
72
73 # IPython modules
73 # IPython modules
74 from IPython.core.splitinput import split_user_input, LineInfo
74 from IPython.core.splitinput import split_user_input, LineInfo
75 from IPython.utils.py3compat import cast_unicode
75 from IPython.utils.py3compat import cast_unicode
76 from IPython.core.inputtransformer import (leading_indent,
76 from IPython.core.inputtransformer import (leading_indent,
77 classic_prompt,
77 classic_prompt,
78 ipy_prompt,
78 ipy_prompt,
79 cellmagic,
79 cellmagic,
80 help_end,
80 help_end,
81 escaped_transformer,
81 escaped_transformer,
82 assign_from_magic,
82 assign_from_magic,
83 assign_from_system,
83 assign_from_system,
84 )
84 )
85
85
86 # Temporary!
86 # Temporary!
87 from IPython.core.inputtransformer import (ESC_SHELL, ESC_SH_CAP, ESC_HELP,
87 from IPython.core.inputtransformer import (ESC_SHELL, ESC_SH_CAP, ESC_HELP,
88 ESC_HELP2, ESC_MAGIC, ESC_MAGIC2,
88 ESC_HELP2, ESC_MAGIC, ESC_MAGIC2,
89 ESC_QUOTE, ESC_QUOTE2, ESC_PAREN, ESC_SEQUENCES)
89 ESC_QUOTE, ESC_QUOTE2, ESC_PAREN, ESC_SEQUENCES)
90
90
91 #-----------------------------------------------------------------------------
91 #-----------------------------------------------------------------------------
92 # Utilities
92 # Utilities
93 #-----------------------------------------------------------------------------
93 #-----------------------------------------------------------------------------
94
94
95 # FIXME: These are general-purpose utilities that later can be moved to the
95 # FIXME: These are general-purpose utilities that later can be moved to the
96 # general ward. Kept here for now because we're being very strict about test
96 # general ward. Kept here for now because we're being very strict about test
97 # coverage with this code, and this lets us ensure that we keep 100% coverage
97 # coverage with this code, and this lets us ensure that we keep 100% coverage
98 # while developing.
98 # while developing.
99
99
100 # compiled regexps for autoindent management
100 # compiled regexps for autoindent management
101 dedent_re = re.compile('|'.join([
101 dedent_re = re.compile('|'.join([
102 r'^\s+raise(\s.*)?$', # raise statement (+ space + other stuff, maybe)
102 r'^\s+raise(\s.*)?$', # raise statement (+ space + other stuff, maybe)
103 r'^\s+raise\([^\)]*\).*$', # wacky raise with immediate open paren
103 r'^\s+raise\([^\)]*\).*$', # wacky raise with immediate open paren
104 r'^\s+return(\s.*)?$', # normal return (+ space + other stuff, maybe)
104 r'^\s+return(\s.*)?$', # normal return (+ space + other stuff, maybe)
105 r'^\s+return\([^\)]*\).*$', # wacky return with immediate open paren
105 r'^\s+return\([^\)]*\).*$', # wacky return with immediate open paren
106 r'^\s+pass\s*$', # pass (optionally followed by trailing spaces)
106 r'^\s+pass\s*$', # pass (optionally followed by trailing spaces)
107 r'^\s+break\s*$', # break (optionally followed by trailing spaces)
107 r'^\s+break\s*$', # break (optionally followed by trailing spaces)
108 r'^\s+continue\s*$', # continue (optionally followed by trailing spaces)
108 r'^\s+continue\s*$', # continue (optionally followed by trailing spaces)
109 ]))
109 ]))
110 ini_spaces_re = re.compile(r'^([ \t\r\f\v]+)')
110 ini_spaces_re = re.compile(r'^([ \t\r\f\v]+)')
111
111
112 # regexp to match pure comment lines so we don't accidentally insert 'if 1:'
112 # regexp to match pure comment lines so we don't accidentally insert 'if 1:'
113 # before pure comments
113 # before pure comments
114 comment_line_re = re.compile('^\s*\#')
114 comment_line_re = re.compile('^\s*\#')
115
115
116
116
117 def num_ini_spaces(s):
117 def num_ini_spaces(s):
118 """Return the number of initial spaces in a string.
118 """Return the number of initial spaces in a string.
119
119
120 Note that tabs are counted as a single space. For now, we do *not* support
120 Note that tabs are counted as a single space. For now, we do *not* support
121 mixing of tabs and spaces in the user's input.
121 mixing of tabs and spaces in the user's input.
122
122
123 Parameters
123 Parameters
124 ----------
124 ----------
125 s : string
125 s : string
126
126
127 Returns
127 Returns
128 -------
128 -------
129 n : int
129 n : int
130 """
130 """
131
131
132 ini_spaces = ini_spaces_re.match(s)
132 ini_spaces = ini_spaces_re.match(s)
133 if ini_spaces:
133 if ini_spaces:
134 return ini_spaces.end()
134 return ini_spaces.end()
135 else:
135 else:
136 return 0
136 return 0
137
137
138 def last_blank(src):
138 def last_blank(src):
139 """Determine if the input source ends in a blank.
139 """Determine if the input source ends in a blank.
140
140
141 A blank is either a newline or a line consisting of whitespace.
141 A blank is either a newline or a line consisting of whitespace.
142
142
143 Parameters
143 Parameters
144 ----------
144 ----------
145 src : string
145 src : string
146 A single or multiline string.
146 A single or multiline string.
147 """
147 """
148 if not src: return False
148 if not src: return False
149 ll = src.splitlines()[-1]
149 ll = src.splitlines()[-1]
150 return (ll == '') or ll.isspace()
150 return (ll == '') or ll.isspace()
151
151
152
152
153 last_two_blanks_re = re.compile(r'\n\s*\n\s*$', re.MULTILINE)
153 last_two_blanks_re = re.compile(r'\n\s*\n\s*$', re.MULTILINE)
154 last_two_blanks_re2 = re.compile(r'.+\n\s*\n\s+$', re.MULTILINE)
154 last_two_blanks_re2 = re.compile(r'.+\n\s*\n\s+$', re.MULTILINE)
155
155
156 def last_two_blanks(src):
156 def last_two_blanks(src):
157 """Determine if the input source ends in two blanks.
157 """Determine if the input source ends in two blanks.
158
158
159 A blank is either a newline or a line consisting of whitespace.
159 A blank is either a newline or a line consisting of whitespace.
160
160
161 Parameters
161 Parameters
162 ----------
162 ----------
163 src : string
163 src : string
164 A single or multiline string.
164 A single or multiline string.
165 """
165 """
166 if not src: return False
166 if not src: return False
167 # The logic here is tricky: I couldn't get a regexp to work and pass all
167 # The logic here is tricky: I couldn't get a regexp to work and pass all
168 # the tests, so I took a different approach: split the source by lines,
168 # the tests, so I took a different approach: split the source by lines,
169 # grab the last two and prepend '###\n' as a stand-in for whatever was in
169 # grab the last two and prepend '###\n' as a stand-in for whatever was in
170 # the body before the last two lines. Then, with that structure, it's
170 # the body before the last two lines. Then, with that structure, it's
171 # possible to analyze with two regexps. Not the most elegant solution, but
171 # possible to analyze with two regexps. Not the most elegant solution, but
172 # it works. If anyone tries to change this logic, make sure to validate
172 # it works. If anyone tries to change this logic, make sure to validate
173 # the whole test suite first!
173 # the whole test suite first!
174 new_src = '\n'.join(['###\n'] + src.splitlines()[-2:])
174 new_src = '\n'.join(['###\n'] + src.splitlines()[-2:])
175 return (bool(last_two_blanks_re.match(new_src)) or
175 return (bool(last_two_blanks_re.match(new_src)) or
176 bool(last_two_blanks_re2.match(new_src)) )
176 bool(last_two_blanks_re2.match(new_src)) )
177
177
178
178
179 def remove_comments(src):
179 def remove_comments(src):
180 """Remove all comments from input source.
180 """Remove all comments from input source.
181
181
182 Note: comments are NOT recognized inside of strings!
182 Note: comments are NOT recognized inside of strings!
183
183
184 Parameters
184 Parameters
185 ----------
185 ----------
186 src : string
186 src : string
187 A single or multiline input string.
187 A single or multiline input string.
188
188
189 Returns
189 Returns
190 -------
190 -------
191 String with all Python comments removed.
191 String with all Python comments removed.
192 """
192 """
193
193
194 return re.sub('#.*', '', src)
194 return re.sub('#.*', '', src)
195
195
196
196
197 def get_input_encoding():
197 def get_input_encoding():
198 """Return the default standard input encoding.
198 """Return the default standard input encoding.
199
199
200 If sys.stdin has no encoding, 'ascii' is returned."""
200 If sys.stdin has no encoding, 'ascii' is returned."""
201 # There are strange environments for which sys.stdin.encoding is None. We
201 # There are strange environments for which sys.stdin.encoding is None. We
202 # ensure that a valid encoding is returned.
202 # ensure that a valid encoding is returned.
203 encoding = getattr(sys.stdin, 'encoding', None)
203 encoding = getattr(sys.stdin, 'encoding', None)
204 if encoding is None:
204 if encoding is None:
205 encoding = 'ascii'
205 encoding = 'ascii'
206 return encoding
206 return encoding
207
207
208 #-----------------------------------------------------------------------------
208 #-----------------------------------------------------------------------------
209 # Classes and functions for normal Python syntax handling
209 # Classes and functions for normal Python syntax handling
210 #-----------------------------------------------------------------------------
210 #-----------------------------------------------------------------------------
211
211
212 class InputSplitter(object):
212 class InputSplitter(object):
213 """An object that can accumulate lines of Python source before execution.
213 """An object that can accumulate lines of Python source before execution.
214
214
215 This object is designed to be fed python source line-by-line, using
215 This object is designed to be fed python source line-by-line, using
216 :meth:`push`. It will return on each push whether the currently pushed
216 :meth:`push`. It will return on each push whether the currently pushed
217 code could be executed already. In addition, it provides a method called
217 code could be executed already. In addition, it provides a method called
218 :meth:`push_accepts_more` that can be used to query whether more input
218 :meth:`push_accepts_more` that can be used to query whether more input
219 can be pushed into a single interactive block.
219 can be pushed into a single interactive block.
220
220
221 This is a simple example of how an interactive terminal-based client can use
221 This is a simple example of how an interactive terminal-based client can use
222 this tool::
222 this tool::
223
223
224 isp = InputSplitter()
224 isp = InputSplitter()
225 while isp.push_accepts_more():
225 while isp.push_accepts_more():
226 indent = ' '*isp.indent_spaces
226 indent = ' '*isp.indent_spaces
227 prompt = '>>> ' + indent
227 prompt = '>>> ' + indent
228 line = indent + raw_input(prompt)
228 line = indent + raw_input(prompt)
229 isp.push(line)
229 isp.push(line)
230 print 'Input source was:\n', isp.source_reset(),
230 print 'Input source was:\n', isp.source_reset(),
231 """
231 """
232 # Number of spaces of indentation computed from input that has been pushed
232 # Number of spaces of indentation computed from input that has been pushed
233 # so far. This is the attributes callers should query to get the current
233 # so far. This is the attributes callers should query to get the current
234 # indentation level, in order to provide auto-indent facilities.
234 # indentation level, in order to provide auto-indent facilities.
235 indent_spaces = 0
235 indent_spaces = 0
236 # String, indicating the default input encoding. It is computed by default
236 # String, indicating the default input encoding. It is computed by default
237 # at initialization time via get_input_encoding(), but it can be reset by a
237 # at initialization time via get_input_encoding(), but it can be reset by a
238 # client with specific knowledge of the encoding.
238 # client with specific knowledge of the encoding.
239 encoding = ''
239 encoding = ''
240 # String where the current full source input is stored, properly encoded.
240 # String where the current full source input is stored, properly encoded.
241 # Reading this attribute is the normal way of querying the currently pushed
241 # Reading this attribute is the normal way of querying the currently pushed
242 # source code, that has been properly encoded.
242 # source code, that has been properly encoded.
243 source = ''
243 source = ''
244 # Code object corresponding to the current source. It is automatically
244 # Code object corresponding to the current source. It is automatically
245 # synced to the source, so it can be queried at any time to obtain the code
245 # synced to the source, so it can be queried at any time to obtain the code
246 # object; it will be None if the source doesn't compile to valid Python.
246 # object; it will be None if the source doesn't compile to valid Python.
247 code = None
247 code = None
248 # Input mode
248 # Input mode
249 input_mode = 'line'
249 input_mode = 'line'
250
250
251 # Private attributes
251 # Private attributes
252
252
253 # List with lines of input accumulated so far
253 # List with lines of input accumulated so far
254 _buffer = None
254 _buffer = None
255 # Command compiler
255 # Command compiler
256 _compile = None
256 _compile = None
257 # Mark when input has changed indentation all the way back to flush-left
257 # Mark when input has changed indentation all the way back to flush-left
258 _full_dedent = False
258 _full_dedent = False
259 # Boolean indicating whether the current block is complete
259 # Boolean indicating whether the current block is complete
260 _is_complete = None
260 _is_complete = None
261
261
262 def __init__(self, input_mode=None):
262 def __init__(self, input_mode=None):
263 """Create a new InputSplitter instance.
263 """Create a new InputSplitter instance.
264
264
265 Parameters
265 Parameters
266 ----------
266 ----------
267 input_mode : str
267 input_mode : str
268
268
269 One of ['line', 'cell']; default is 'line'.
269 One of ['line', 'cell']; default is 'line'.
270
270
271 The input_mode parameter controls how new inputs are used when fed via
271 The input_mode parameter controls how new inputs are used when fed via
272 the :meth:`push` method:
272 the :meth:`push` method:
273
273
274 - 'line': meant for line-oriented clients, inputs are appended one at a
274 - 'line': meant for line-oriented clients, inputs are appended one at a
275 time to the internal buffer and the whole buffer is compiled.
275 time to the internal buffer and the whole buffer is compiled.
276
276
277 - 'cell': meant for clients that can edit multi-line 'cells' of text at
277 - 'cell': meant for clients that can edit multi-line 'cells' of text at
278 a time. A cell can contain one or more blocks that can be compile in
278 a time. A cell can contain one or more blocks that can be compile in
279 'single' mode by Python. In this mode, each new input new input
279 'single' mode by Python. In this mode, each new input new input
280 completely replaces all prior inputs. Cell mode is thus equivalent
280 completely replaces all prior inputs. Cell mode is thus equivalent
281 to prepending a full reset() to every push() call.
281 to prepending a full reset() to every push() call.
282 """
282 """
283 self._buffer = []
283 self._buffer = []
284 self._compile = codeop.CommandCompiler()
284 self._compile = codeop.CommandCompiler()
285 self.encoding = get_input_encoding()
285 self.encoding = get_input_encoding()
286 self.input_mode = InputSplitter.input_mode if input_mode is None \
286 self.input_mode = InputSplitter.input_mode if input_mode is None \
287 else input_mode
287 else input_mode
288
288
289 def reset(self):
289 def reset(self):
290 """Reset the input buffer and associated state."""
290 """Reset the input buffer and associated state."""
291 self.indent_spaces = 0
291 self.indent_spaces = 0
292 self._buffer[:] = []
292 self._buffer[:] = []
293 self.source = ''
293 self.source = ''
294 self.code = None
294 self.code = None
295 self._is_complete = False
295 self._is_complete = False
296 self._full_dedent = False
296 self._full_dedent = False
297
297
298 def source_reset(self):
298 def source_reset(self):
299 """Return the input source and perform a full reset.
299 """Return the input source and perform a full reset.
300 """
300 """
301 out = self.source
301 out = self.source
302 self.reset()
302 self.reset()
303 return out
303 return out
304
304
305 def push(self, lines):
305 def push(self, lines):
306 """Push one or more lines of input.
306 """Push one or more lines of input.
307
307
308 This stores the given lines and returns a status code indicating
308 This stores the given lines and returns a status code indicating
309 whether the code forms a complete Python block or not.
309 whether the code forms a complete Python block or not.
310
310
311 Any exceptions generated in compilation are swallowed, but if an
311 Any exceptions generated in compilation are swallowed, but if an
312 exception was produced, the method returns True.
312 exception was produced, the method returns True.
313
313
314 Parameters
314 Parameters
315 ----------
315 ----------
316 lines : string
316 lines : string
317 One or more lines of Python input.
317 One or more lines of Python input.
318
318
319 Returns
319 Returns
320 -------
320 -------
321 is_complete : boolean
321 is_complete : boolean
322 True if the current input source (the result of the current input
322 True if the current input source (the result of the current input
323 plus prior inputs) forms a complete Python execution block. Note that
323 plus prior inputs) forms a complete Python execution block. Note that
324 this value is also stored as a private attribute (``_is_complete``), so it
324 this value is also stored as a private attribute (``_is_complete``), so it
325 can be queried at any time.
325 can be queried at any time.
326 """
326 """
327 if self.input_mode == 'cell':
327 if self.input_mode == 'cell':
328 self.reset()
328 self.reset()
329
329
330 self._store(lines)
330 self._store(lines)
331 source = self.source
331 source = self.source
332
332
333 # Before calling _compile(), reset the code object to None so that if an
333 # Before calling _compile(), reset the code object to None so that if an
334 # exception is raised in compilation, we don't mislead by having
334 # exception is raised in compilation, we don't mislead by having
335 # inconsistent code/source attributes.
335 # inconsistent code/source attributes.
336 self.code, self._is_complete = None, None
336 self.code, self._is_complete = None, None
337
337
338 # Honor termination lines properly
338 # Honor termination lines properly
339 if source.endswith('\\\n'):
339 if source.endswith('\\\n'):
340 return False
340 return False
341
341
342 self._update_indent(lines)
342 self._update_indent(lines)
343 try:
343 try:
344 self.code = self._compile(source, symbol="exec")
344 self.code = self._compile(source, symbol="exec")
345 # Invalid syntax can produce any of a number of different errors from
345 # Invalid syntax can produce any of a number of different errors from
346 # inside the compiler, so we have to catch them all. Syntax errors
346 # inside the compiler, so we have to catch them all. Syntax errors
347 # immediately produce a 'ready' block, so the invalid Python can be
347 # immediately produce a 'ready' block, so the invalid Python can be
348 # sent to the kernel for evaluation with possible ipython
348 # sent to the kernel for evaluation with possible ipython
349 # special-syntax conversion.
349 # special-syntax conversion.
350 except (SyntaxError, OverflowError, ValueError, TypeError,
350 except (SyntaxError, OverflowError, ValueError, TypeError,
351 MemoryError):
351 MemoryError):
352 self._is_complete = True
352 self._is_complete = True
353 else:
353 else:
354 # Compilation didn't produce any exceptions (though it may not have
354 # Compilation didn't produce any exceptions (though it may not have
355 # given a complete code object)
355 # given a complete code object)
356 self._is_complete = self.code is not None
356 self._is_complete = self.code is not None
357
357
358 return self._is_complete
358 return self._is_complete
359
359
360 def push_accepts_more(self):
360 def push_accepts_more(self):
361 """Return whether a block of interactive input can accept more input.
361 """Return whether a block of interactive input can accept more input.
362
362
363 This method is meant to be used by line-oriented frontends, who need to
363 This method is meant to be used by line-oriented frontends, who need to
364 guess whether a block is complete or not based solely on prior and
364 guess whether a block is complete or not based solely on prior and
365 current input lines. The InputSplitter considers it has a complete
365 current input lines. The InputSplitter considers it has a complete
366 interactive block and will not accept more input only when either a
366 interactive block and will not accept more input only when either a
367 SyntaxError is raised, or *all* of the following are true:
367 SyntaxError is raised, or *all* of the following are true:
368
368
369 1. The input compiles to a complete statement.
369 1. The input compiles to a complete statement.
370
370
371 2. The indentation level is flush-left (because if we are indented,
371 2. The indentation level is flush-left (because if we are indented,
372 like inside a function definition or for loop, we need to keep
372 like inside a function definition or for loop, we need to keep
373 reading new input).
373 reading new input).
374
374
375 3. There is one extra line consisting only of whitespace.
375 3. There is one extra line consisting only of whitespace.
376
376
377 Because of condition #3, this method should be used only by
377 Because of condition #3, this method should be used only by
378 *line-oriented* frontends, since it means that intermediate blank lines
378 *line-oriented* frontends, since it means that intermediate blank lines
379 are not allowed in function definitions (or any other indented block).
379 are not allowed in function definitions (or any other indented block).
380
380
381 If the current input produces a syntax error, this method immediately
381 If the current input produces a syntax error, this method immediately
382 returns False but does *not* raise the syntax error exception, as
382 returns False but does *not* raise the syntax error exception, as
383 typically clients will want to send invalid syntax to an execution
383 typically clients will want to send invalid syntax to an execution
384 backend which might convert the invalid syntax into valid Python via
384 backend which might convert the invalid syntax into valid Python via
385 one of the dynamic IPython mechanisms.
385 one of the dynamic IPython mechanisms.
386 """
386 """
387
387
388 # With incomplete input, unconditionally accept more
388 # With incomplete input, unconditionally accept more
389 if not self._is_complete:
389 if not self._is_complete:
390 return True
390 return True
391
391
392 # If we already have complete input and we're flush left, the answer
392 # If we already have complete input and we're flush left, the answer
393 # depends. In line mode, if there hasn't been any indentation,
393 # depends. In line mode, if there hasn't been any indentation,
394 # that's it. If we've come back from some indentation, we need
394 # that's it. If we've come back from some indentation, we need
395 # the blank final line to finish.
395 # the blank final line to finish.
396 # In cell mode, we need to check how many blocks the input so far
396 # In cell mode, we need to check how many blocks the input so far
397 # compiles into, because if there's already more than one full
397 # compiles into, because if there's already more than one full
398 # independent block of input, then the client has entered full
398 # independent block of input, then the client has entered full
399 # 'cell' mode and is feeding lines that each is complete. In this
399 # 'cell' mode and is feeding lines that each is complete. In this
400 # case we should then keep accepting. The Qt terminal-like console
400 # case we should then keep accepting. The Qt terminal-like console
401 # does precisely this, to provide the convenience of terminal-like
401 # does precisely this, to provide the convenience of terminal-like
402 # input of single expressions, but allowing the user (with a
402 # input of single expressions, but allowing the user (with a
403 # separate keystroke) to switch to 'cell' mode and type multiple
403 # separate keystroke) to switch to 'cell' mode and type multiple
404 # expressions in one shot.
404 # expressions in one shot.
405 if self.indent_spaces==0:
405 if self.indent_spaces==0:
406 if self.input_mode=='line':
406 if self.input_mode=='line':
407 if not self._full_dedent:
407 if not self._full_dedent:
408 return False
408 return False
409 else:
409 else:
410 try:
410 try:
411 code_ast = ast.parse(u''.join(self._buffer))
411 code_ast = ast.parse(u''.join(self._buffer))
412 except Exception:
412 except Exception:
413 return False
413 return False
414 else:
414 else:
415 if len(code_ast.body) == 1:
415 if len(code_ast.body) == 1:
416 return False
416 return False
417
417
418 # When input is complete, then termination is marked by an extra blank
418 # When input is complete, then termination is marked by an extra blank
419 # line at the end.
419 # line at the end.
420 last_line = self.source.splitlines()[-1]
420 last_line = self.source.splitlines()[-1]
421 return bool(last_line and not last_line.isspace())
421 return bool(last_line and not last_line.isspace())
422
422
423 #------------------------------------------------------------------------
423 #------------------------------------------------------------------------
424 # Private interface
424 # Private interface
425 #------------------------------------------------------------------------
425 #------------------------------------------------------------------------
426
426
427 def _find_indent(self, line):
427 def _find_indent(self, line):
428 """Compute the new indentation level for a single line.
428 """Compute the new indentation level for a single line.
429
429
430 Parameters
430 Parameters
431 ----------
431 ----------
432 line : str
432 line : str
433 A single new line of non-whitespace, non-comment Python input.
433 A single new line of non-whitespace, non-comment Python input.
434
434
435 Returns
435 Returns
436 -------
436 -------
437 indent_spaces : int
437 indent_spaces : int
438 New value for the indent level (it may be equal to self.indent_spaces
438 New value for the indent level (it may be equal to self.indent_spaces
439 if indentation doesn't change.
439 if indentation doesn't change.
440
440
441 full_dedent : boolean
441 full_dedent : boolean
442 Whether the new line causes a full flush-left dedent.
442 Whether the new line causes a full flush-left dedent.
443 """
443 """
444 indent_spaces = self.indent_spaces
444 indent_spaces = self.indent_spaces
445 full_dedent = self._full_dedent
445 full_dedent = self._full_dedent
446
446
447 inisp = num_ini_spaces(line)
447 inisp = num_ini_spaces(line)
448 if inisp < indent_spaces:
448 if inisp < indent_spaces:
449 indent_spaces = inisp
449 indent_spaces = inisp
450 if indent_spaces <= 0:
450 if indent_spaces <= 0:
451 #print 'Full dedent in text',self.source # dbg
451 #print 'Full dedent in text',self.source # dbg
452 full_dedent = True
452 full_dedent = True
453
453
454 if line.rstrip()[-1] == ':':
454 if line.rstrip()[-1] == ':':
455 indent_spaces += 4
455 indent_spaces += 4
456 elif dedent_re.match(line):
456 elif dedent_re.match(line):
457 indent_spaces -= 4
457 indent_spaces -= 4
458 if indent_spaces <= 0:
458 if indent_spaces <= 0:
459 full_dedent = True
459 full_dedent = True
460
460
461 # Safety
461 # Safety
462 if indent_spaces < 0:
462 if indent_spaces < 0:
463 indent_spaces = 0
463 indent_spaces = 0
464 #print 'safety' # dbg
464 #print 'safety' # dbg
465
465
466 return indent_spaces, full_dedent
466 return indent_spaces, full_dedent
467
467
468 def _update_indent(self, lines):
468 def _update_indent(self, lines):
469 for line in remove_comments(lines).splitlines():
469 for line in remove_comments(lines).splitlines():
470 if line and not line.isspace():
470 if line and not line.isspace():
471 self.indent_spaces, self._full_dedent = self._find_indent(line)
471 self.indent_spaces, self._full_dedent = self._find_indent(line)
472
472
473 def _store(self, lines, buffer=None, store='source'):
473 def _store(self, lines, buffer=None, store='source'):
474 """Store one or more lines of input.
474 """Store one or more lines of input.
475
475
476 If input lines are not newline-terminated, a newline is automatically
476 If input lines are not newline-terminated, a newline is automatically
477 appended."""
477 appended."""
478
478
479 if buffer is None:
479 if buffer is None:
480 buffer = self._buffer
480 buffer = self._buffer
481
481
482 if lines.endswith('\n'):
482 if lines.endswith('\n'):
483 buffer.append(lines)
483 buffer.append(lines)
484 else:
484 else:
485 buffer.append(lines+'\n')
485 buffer.append(lines+'\n')
486 setattr(self, store, self._set_source(buffer))
486 setattr(self, store, self._set_source(buffer))
487
487
488 def _set_source(self, buffer):
488 def _set_source(self, buffer):
489 return u''.join(buffer)
489 return u''.join(buffer)
490
490
491
491
492 class IPythonInputSplitter(InputSplitter):
492 class IPythonInputSplitter(InputSplitter):
493 """An input splitter that recognizes all of IPython's special syntax."""
493 """An input splitter that recognizes all of IPython's special syntax."""
494
494
495 # String with raw, untransformed input.
495 # String with raw, untransformed input.
496 source_raw = ''
496 source_raw = ''
497
497
498 # Flag to track when a transformer has stored input that it hasn't given
498 # Flag to track when a transformer has stored input that it hasn't given
499 # back yet.
499 # back yet.
500 transformer_accumulating = False
500 transformer_accumulating = False
501
501
502 # Private attributes
502 # Private attributes
503
503
504 # List with lines of raw input accumulated so far.
504 # List with lines of raw input accumulated so far.
505 _buffer_raw = None
505 _buffer_raw = None
506
506
507 def __init__(self, input_mode=None):
507 def __init__(self, input_mode=None):
508 super(IPythonInputSplitter, self).__init__(input_mode)
508 super(IPythonInputSplitter, self).__init__(input_mode)
509 self._buffer_raw = []
509 self._buffer_raw = []
510 self._validate = True
510 self._validate = True
511 self.transforms = [leading_indent,
511 self.transforms = [leading_indent,
512 classic_prompt,
512 classic_prompt,
513 ipy_prompt,
513 ipy_prompt,
514 cellmagic,
514 cellmagic,
515 help_end,
515 help_end,
516 escaped_transformer,
516 escaped_transformer,
517 assign_from_magic,
517 assign_from_magic,
518 assign_from_system,
518 assign_from_system,
519 ]
519 ]
520
520
521 def reset(self):
521 def reset(self):
522 """Reset the input buffer and associated state."""
522 """Reset the input buffer and associated state."""
523 super(IPythonInputSplitter, self).reset()
523 super(IPythonInputSplitter, self).reset()
524 self._buffer_raw[:] = []
524 self._buffer_raw[:] = []
525 self.source_raw = ''
525 self.source_raw = ''
526 self.transformer_accumulating = False
526 self.transformer_accumulating = False
527 for t in self.transforms:
528 t.reset()
529
530 def flush_transformers(self):
531 out = None
532 for t in self.transforms:
533 tmp = t.reset()
534 if tmp:
535 out = tmp
536 if out:
537 self._store(out)
527
538
528 def source_raw_reset(self):
539 def source_raw_reset(self):
529 """Return input and raw source and perform a full reset.
540 """Return input and raw source and perform a full reset.
530 """
541 """
542 self.flush_transformers()
531 out = self.source
543 out = self.source
532 out_r = self.source_raw
544 out_r = self.source_raw
533 self.reset()
545 self.reset()
534 return out, out_r
546 return out, out_r
535
547
548 def source_reset(self):
549 self.flush_transformers()
550 return super(IPythonInputSplitter, self).source_reset()
551
536 def push_accepts_more(self):
552 def push_accepts_more(self):
537 if self.transformer_accumulating:
553 if self.transformer_accumulating:
538 return True
554 return True
539 else:
555 else:
540 return super(IPythonInputSplitter, self).push_accepts_more()
556 return super(IPythonInputSplitter, self).push_accepts_more()
541
557
542 def transform_cell(self, cell):
558 def transform_cell(self, cell):
543 """Process and translate a cell of input.
559 """Process and translate a cell of input.
544 """
560 """
545 self.reset()
561 self.reset()
546 self.push(cell)
562 self.push(cell)
547 return self.source_reset()
563 return self.source_reset()
548
564
549 def push(self, lines):
565 def push(self, lines):
550 """Push one or more lines of IPython input.
566 """Push one or more lines of IPython input.
551
567
552 This stores the given lines and returns a status code indicating
568 This stores the given lines and returns a status code indicating
553 whether the code forms a complete Python block or not, after processing
569 whether the code forms a complete Python block or not, after processing
554 all input lines for special IPython syntax.
570 all input lines for special IPython syntax.
555
571
556 Any exceptions generated in compilation are swallowed, but if an
572 Any exceptions generated in compilation are swallowed, but if an
557 exception was produced, the method returns True.
573 exception was produced, the method returns True.
558
574
559 Parameters
575 Parameters
560 ----------
576 ----------
561 lines : string
577 lines : string
562 One or more lines of Python input.
578 One or more lines of Python input.
563
579
564 Returns
580 Returns
565 -------
581 -------
566 is_complete : boolean
582 is_complete : boolean
567 True if the current input source (the result of the current input
583 True if the current input source (the result of the current input
568 plus prior inputs) forms a complete Python execution block. Note that
584 plus prior inputs) forms a complete Python execution block. Note that
569 this value is also stored as a private attribute (_is_complete), so it
585 this value is also stored as a private attribute (_is_complete), so it
570 can be queried at any time.
586 can be queried at any time.
571 """
587 """
572
588
573 # We must ensure all input is pure unicode
589 # We must ensure all input is pure unicode
574 lines = cast_unicode(lines, self.encoding)
590 lines = cast_unicode(lines, self.encoding)
575
591
576 # ''.splitlines() --> [], but we need to push the empty line to transformers
592 # ''.splitlines() --> [], but we need to push the empty line to transformers
577 lines_list = lines.splitlines()
593 lines_list = lines.splitlines()
578 if not lines_list:
594 if not lines_list:
579 lines_list = ['']
595 lines_list = ['']
580
596
581 # Transform logic
597 # Transform logic
582 #
598 #
583 # We only apply the line transformers to the input if we have either no
599 # We only apply the line transformers to the input if we have either no
584 # input yet, or complete input, or if the last line of the buffer ends
600 # input yet, or complete input, or if the last line of the buffer ends
585 # with ':' (opening an indented block). This prevents the accidental
601 # with ':' (opening an indented block). This prevents the accidental
586 # transformation of escapes inside multiline expressions like
602 # transformation of escapes inside multiline expressions like
587 # triple-quoted strings or parenthesized expressions.
603 # triple-quoted strings or parenthesized expressions.
588 #
604 #
589 # The last heuristic, while ugly, ensures that the first line of an
605 # The last heuristic, while ugly, ensures that the first line of an
590 # indented block is correctly transformed.
606 # indented block is correctly transformed.
591 #
607 #
592 # FIXME: try to find a cleaner approach for this last bit.
608 # FIXME: try to find a cleaner approach for this last bit.
593
609
594 # If we were in 'block' mode, since we're going to pump the parent
610 # If we were in 'block' mode, since we're going to pump the parent
595 # class by hand line by line, we need to temporarily switch out to
611 # class by hand line by line, we need to temporarily switch out to
596 # 'line' mode, do a single manual reset and then feed the lines one
612 # 'line' mode, do a single manual reset and then feed the lines one
597 # by one. Note that this only matters if the input has more than one
613 # by one. Note that this only matters if the input has more than one
598 # line.
614 # line.
599 changed_input_mode = False
615 changed_input_mode = False
600
616
601 if self.input_mode == 'cell':
617 if self.input_mode == 'cell':
602 self.reset()
618 self.reset()
603 changed_input_mode = True
619 changed_input_mode = True
604 saved_input_mode = 'cell'
620 saved_input_mode = 'cell'
605 self.input_mode = 'line'
621 self.input_mode = 'line'
606
622
607 # Store raw source before applying any transformations to it. Note
623 # Store raw source before applying any transformations to it. Note
608 # that this must be done *after* the reset() call that would otherwise
624 # that this must be done *after* the reset() call that would otherwise
609 # flush the buffer.
625 # flush the buffer.
610 self._store(lines, self._buffer_raw, 'source_raw')
626 self._store(lines, self._buffer_raw, 'source_raw')
611
627
612 try:
628 try:
613 for line in lines_list:
629 for line in lines_list:
614 out = self.push_line(line)
630 out = self.push_line(line)
615 finally:
631 finally:
616 if changed_input_mode:
632 if changed_input_mode:
617 self.input_mode = saved_input_mode
633 self.input_mode = saved_input_mode
618
634
619 return out
635 return out
620
636
621 def push_line(self, line):
637 def push_line(self, line):
622 buf = self._buffer
638 buf = self._buffer
623 not_in_string = self._is_complete or not buf or \
639 not_in_string = self._is_complete or not buf or \
624 (buf and buf[-1].rstrip().endswith((':', ',')))
640 (buf and buf[-1].rstrip().endswith((':', ',')))
625 for transformer in self.transforms:
641 for transformer in self.transforms:
626 if not_in_string or transformer.look_in_string:
642 if not_in_string or transformer.look_in_string:
627 line = transformer.push(line)
643 line = transformer.push(line)
628 if line is None:
644 if line is None:
629 self.transformer_accumulating = True
645 self.transformer_accumulating = True
630 return False
646 return False
631
647
632 self.transformer_accumulating = False
648 self.transformer_accumulating = False
633 return super(IPythonInputSplitter, self).push(line)
649 return super(IPythonInputSplitter, self).push(line)
@@ -1,323 +1,323 b''
1 import abc
1 import abc
2 import re
2 import re
3 from StringIO import StringIO
3 from StringIO import StringIO
4 import tokenize
4 import tokenize
5
5
6 from IPython.core.splitinput import split_user_input, LineInfo
6 from IPython.core.splitinput import split_user_input, LineInfo
7
7
8 #-----------------------------------------------------------------------------
8 #-----------------------------------------------------------------------------
9 # Globals
9 # Globals
10 #-----------------------------------------------------------------------------
10 #-----------------------------------------------------------------------------
11
11
12 # The escape sequences that define the syntax transformations IPython will
12 # The escape sequences that define the syntax transformations IPython will
13 # apply to user input. These can NOT be just changed here: many regular
13 # apply to user input. These can NOT be just changed here: many regular
14 # expressions and other parts of the code may use their hardcoded values, and
14 # expressions and other parts of the code may use their hardcoded values, and
15 # for all intents and purposes they constitute the 'IPython syntax', so they
15 # for all intents and purposes they constitute the 'IPython syntax', so they
16 # should be considered fixed.
16 # should be considered fixed.
17
17
18 ESC_SHELL = '!' # Send line to underlying system shell
18 ESC_SHELL = '!' # Send line to underlying system shell
19 ESC_SH_CAP = '!!' # Send line to system shell and capture output
19 ESC_SH_CAP = '!!' # Send line to system shell and capture output
20 ESC_HELP = '?' # Find information about object
20 ESC_HELP = '?' # Find information about object
21 ESC_HELP2 = '??' # Find extra-detailed information about object
21 ESC_HELP2 = '??' # Find extra-detailed information about object
22 ESC_MAGIC = '%' # Call magic function
22 ESC_MAGIC = '%' # Call magic function
23 ESC_MAGIC2 = '%%' # Call cell-magic function
23 ESC_MAGIC2 = '%%' # Call cell-magic function
24 ESC_QUOTE = ',' # Split args on whitespace, quote each as string and call
24 ESC_QUOTE = ',' # Split args on whitespace, quote each as string and call
25 ESC_QUOTE2 = ';' # Quote all args as a single string, call
25 ESC_QUOTE2 = ';' # Quote all args as a single string, call
26 ESC_PAREN = '/' # Call first argument with rest of line as arguments
26 ESC_PAREN = '/' # Call first argument with rest of line as arguments
27
27
28 ESC_SEQUENCES = [ESC_SHELL, ESC_SH_CAP, ESC_HELP ,\
28 ESC_SEQUENCES = [ESC_SHELL, ESC_SH_CAP, ESC_HELP ,\
29 ESC_HELP2, ESC_MAGIC, ESC_MAGIC2,\
29 ESC_HELP2, ESC_MAGIC, ESC_MAGIC2,\
30 ESC_QUOTE, ESC_QUOTE2, ESC_PAREN ]
30 ESC_QUOTE, ESC_QUOTE2, ESC_PAREN ]
31
31
32
32
33
33
34 class InputTransformer(object):
34 class InputTransformer(object):
35 __metaclass__ = abc.ABCMeta
35 __metaclass__ = abc.ABCMeta
36
36
37 @abc.abstractmethod
37 @abc.abstractmethod
38 def push(self, line):
38 def push(self, line):
39 pass
39 pass
40
40
41 @abc.abstractmethod
41 @abc.abstractmethod
42 def reset(self):
42 def reset(self):
43 pass
43 pass
44
44
45 look_in_string = False
45 look_in_string = False
46
46
47 class StatelessInputTransformer(InputTransformer):
47 class StatelessInputTransformer(InputTransformer):
48 """Decorator for a stateless input transformer implemented as a function."""
48 """Decorator for a stateless input transformer implemented as a function."""
49 def __init__(self, func):
49 def __init__(self, func):
50 self.func = func
50 self.func = func
51
51
52 def push(self, line):
52 def push(self, line):
53 return self.func(line)
53 return self.func(line)
54
54
55 def reset(self):
55 def reset(self):
56 pass
56 pass
57
57
58 class CoroutineInputTransformer(InputTransformer):
58 class CoroutineInputTransformer(InputTransformer):
59 """Decorator for an input transformer implemented as a coroutine."""
59 """Decorator for an input transformer implemented as a coroutine."""
60 def __init__(self, coro):
60 def __init__(self, coro):
61 # Prime it
61 # Prime it
62 self.coro = coro()
62 self.coro = coro()
63 next(self.coro)
63 next(self.coro)
64
64
65 def push(self, line):
65 def push(self, line):
66 return self.coro.send(line)
66 return self.coro.send(line)
67
67
68 def reset(self):
68 def reset(self):
69 self.coro.send(None)
69 return self.coro.send(None)
70
70
71
71
72 # Utilities
72 # Utilities
73 def _make_help_call(target, esc, lspace, next_input=None):
73 def _make_help_call(target, esc, lspace, next_input=None):
74 """Prepares a pinfo(2)/psearch call from a target name and the escape
74 """Prepares a pinfo(2)/psearch call from a target name and the escape
75 (i.e. ? or ??)"""
75 (i.e. ? or ??)"""
76 method = 'pinfo2' if esc == '??' \
76 method = 'pinfo2' if esc == '??' \
77 else 'psearch' if '*' in target \
77 else 'psearch' if '*' in target \
78 else 'pinfo'
78 else 'pinfo'
79 arg = " ".join([method, target])
79 arg = " ".join([method, target])
80 if next_input is None:
80 if next_input is None:
81 return '%sget_ipython().magic(%r)' % (lspace, arg)
81 return '%sget_ipython().magic(%r)' % (lspace, arg)
82 else:
82 else:
83 return '%sget_ipython().set_next_input(%r);get_ipython().magic(%r)' % \
83 return '%sget_ipython().set_next_input(%r);get_ipython().magic(%r)' % \
84 (lspace, next_input, arg)
84 (lspace, next_input, arg)
85
85
86 @CoroutineInputTransformer
86 @CoroutineInputTransformer
87 def escaped_transformer():
87 def escaped_transformer():
88 """Translate lines beginning with one of IPython's escape characters."""
88 """Translate lines beginning with one of IPython's escape characters."""
89
89
90 # These define the transformations for the different escape characters.
90 # These define the transformations for the different escape characters.
91 def _tr_system(line_info):
91 def _tr_system(line_info):
92 "Translate lines escaped with: !"
92 "Translate lines escaped with: !"
93 cmd = line_info.line.lstrip().lstrip(ESC_SHELL)
93 cmd = line_info.line.lstrip().lstrip(ESC_SHELL)
94 return '%sget_ipython().system(%r)' % (line_info.pre, cmd)
94 return '%sget_ipython().system(%r)' % (line_info.pre, cmd)
95
95
96 def _tr_system2(line_info):
96 def _tr_system2(line_info):
97 "Translate lines escaped with: !!"
97 "Translate lines escaped with: !!"
98 cmd = line_info.line.lstrip()[2:]
98 cmd = line_info.line.lstrip()[2:]
99 return '%sget_ipython().getoutput(%r)' % (line_info.pre, cmd)
99 return '%sget_ipython().getoutput(%r)' % (line_info.pre, cmd)
100
100
101 def _tr_help(line_info):
101 def _tr_help(line_info):
102 "Translate lines escaped with: ?/??"
102 "Translate lines escaped with: ?/??"
103 # A naked help line should just fire the intro help screen
103 # A naked help line should just fire the intro help screen
104 if not line_info.line[1:]:
104 if not line_info.line[1:]:
105 return 'get_ipython().show_usage()'
105 return 'get_ipython().show_usage()'
106
106
107 return _make_help_call(line_info.ifun, line_info.esc, line_info.pre)
107 return _make_help_call(line_info.ifun, line_info.esc, line_info.pre)
108
108
109 def _tr_magic(line_info):
109 def _tr_magic(line_info):
110 "Translate lines escaped with: %"
110 "Translate lines escaped with: %"
111 tpl = '%sget_ipython().magic(%r)'
111 tpl = '%sget_ipython().magic(%r)'
112 cmd = ' '.join([line_info.ifun, line_info.the_rest]).strip()
112 cmd = ' '.join([line_info.ifun, line_info.the_rest]).strip()
113 return tpl % (line_info.pre, cmd)
113 return tpl % (line_info.pre, cmd)
114
114
115 def _tr_quote(line_info):
115 def _tr_quote(line_info):
116 "Translate lines escaped with: ,"
116 "Translate lines escaped with: ,"
117 return '%s%s("%s")' % (line_info.pre, line_info.ifun,
117 return '%s%s("%s")' % (line_info.pre, line_info.ifun,
118 '", "'.join(line_info.the_rest.split()) )
118 '", "'.join(line_info.the_rest.split()) )
119
119
120 def _tr_quote2(line_info):
120 def _tr_quote2(line_info):
121 "Translate lines escaped with: ;"
121 "Translate lines escaped with: ;"
122 return '%s%s("%s")' % (line_info.pre, line_info.ifun,
122 return '%s%s("%s")' % (line_info.pre, line_info.ifun,
123 line_info.the_rest)
123 line_info.the_rest)
124
124
125 def _tr_paren(line_info):
125 def _tr_paren(line_info):
126 "Translate lines escaped with: /"
126 "Translate lines escaped with: /"
127 return '%s%s(%s)' % (line_info.pre, line_info.ifun,
127 return '%s%s(%s)' % (line_info.pre, line_info.ifun,
128 ", ".join(line_info.the_rest.split()))
128 ", ".join(line_info.the_rest.split()))
129
129
130 tr = { ESC_SHELL : _tr_system,
130 tr = { ESC_SHELL : _tr_system,
131 ESC_SH_CAP : _tr_system2,
131 ESC_SH_CAP : _tr_system2,
132 ESC_HELP : _tr_help,
132 ESC_HELP : _tr_help,
133 ESC_HELP2 : _tr_help,
133 ESC_HELP2 : _tr_help,
134 ESC_MAGIC : _tr_magic,
134 ESC_MAGIC : _tr_magic,
135 ESC_QUOTE : _tr_quote,
135 ESC_QUOTE : _tr_quote,
136 ESC_QUOTE2 : _tr_quote2,
136 ESC_QUOTE2 : _tr_quote2,
137 ESC_PAREN : _tr_paren }
137 ESC_PAREN : _tr_paren }
138
138
139 line = ''
139 line = ''
140 while True:
140 while True:
141 line = (yield line)
141 line = (yield line)
142 if not line or line.isspace():
142 if not line or line.isspace():
143 continue
143 continue
144 lineinf = LineInfo(line)
144 lineinf = LineInfo(line)
145 if lineinf.esc not in tr:
145 if lineinf.esc not in tr:
146 continue
146 continue
147
147
148 parts = []
148 parts = []
149 while line is not None:
149 while line is not None:
150 parts.append(line.rstrip('\\'))
150 parts.append(line.rstrip('\\'))
151 if not line.endswith('\\'):
151 if not line.endswith('\\'):
152 break
152 break
153 line = (yield None)
153 line = (yield None)
154
154
155 # Output
155 # Output
156 lineinf = LineInfo(' '.join(parts))
156 lineinf = LineInfo(' '.join(parts))
157 line = tr[lineinf.esc](lineinf)
157 line = tr[lineinf.esc](lineinf)
158
158
159 _initial_space_re = re.compile(r'\s*')
159 _initial_space_re = re.compile(r'\s*')
160
160
161 _help_end_re = re.compile(r"""(%{0,2}
161 _help_end_re = re.compile(r"""(%{0,2}
162 [a-zA-Z_*][\w*]* # Variable name
162 [a-zA-Z_*][\w*]* # Variable name
163 (\.[a-zA-Z_*][\w*]*)* # .etc.etc
163 (\.[a-zA-Z_*][\w*]*)* # .etc.etc
164 )
164 )
165 (\?\??)$ # ? or ??""",
165 (\?\??)$ # ? or ??""",
166 re.VERBOSE)
166 re.VERBOSE)
167
167
168 def has_comment(src):
168 def has_comment(src):
169 """Indicate whether an input line has (i.e. ends in, or is) a comment.
169 """Indicate whether an input line has (i.e. ends in, or is) a comment.
170
170
171 This uses tokenize, so it can distinguish comments from # inside strings.
171 This uses tokenize, so it can distinguish comments from # inside strings.
172
172
173 Parameters
173 Parameters
174 ----------
174 ----------
175 src : string
175 src : string
176 A single line input string.
176 A single line input string.
177
177
178 Returns
178 Returns
179 -------
179 -------
180 Boolean: True if source has a comment.
180 Boolean: True if source has a comment.
181 """
181 """
182 readline = StringIO(src).readline
182 readline = StringIO(src).readline
183 toktypes = set()
183 toktypes = set()
184 try:
184 try:
185 for t in tokenize.generate_tokens(readline):
185 for t in tokenize.generate_tokens(readline):
186 toktypes.add(t[0])
186 toktypes.add(t[0])
187 except tokenize.TokenError:
187 except tokenize.TokenError:
188 pass
188 pass
189 return(tokenize.COMMENT in toktypes)
189 return(tokenize.COMMENT in toktypes)
190
190
191 @StatelessInputTransformer
191 @StatelessInputTransformer
192 def help_end(line):
192 def help_end(line):
193 """Translate lines with ?/?? at the end"""
193 """Translate lines with ?/?? at the end"""
194 m = _help_end_re.search(line)
194 m = _help_end_re.search(line)
195 if m is None or has_comment(line):
195 if m is None or has_comment(line):
196 return line
196 return line
197 target = m.group(1)
197 target = m.group(1)
198 esc = m.group(3)
198 esc = m.group(3)
199 lspace = _initial_space_re.match(line).group(0)
199 lspace = _initial_space_re.match(line).group(0)
200
200
201 # If we're mid-command, put it back on the next prompt for the user.
201 # If we're mid-command, put it back on the next prompt for the user.
202 next_input = line.rstrip('?') if line.strip() != m.group(0) else None
202 next_input = line.rstrip('?') if line.strip() != m.group(0) else None
203
203
204 return _make_help_call(target, esc, lspace, next_input)
204 return _make_help_call(target, esc, lspace, next_input)
205
205
206
206
207 @CoroutineInputTransformer
207 @CoroutineInputTransformer
208 def cellmagic():
208 def cellmagic():
209 tpl = 'get_ipython().run_cell_magic(%r, %r, %r)'
209 tpl = 'get_ipython().run_cell_magic(%r, %r, %r)'
210 line = ''
210 line = ''
211 while True:
211 while True:
212 line = (yield line)
212 line = (yield line)
213 if (not line) or (not line.startswith(ESC_MAGIC2)):
213 if (not line) or (not line.startswith(ESC_MAGIC2)):
214 continue
214 continue
215
215
216 first = line
216 first = line
217 body = []
217 body = []
218 line = (yield None)
218 line = (yield None)
219 while (line is not None) and (line.strip() != ''):
219 while (line is not None) and (line.strip() != ''):
220 body.append(line)
220 body.append(line)
221 line = (yield None)
221 line = (yield None)
222
222
223 # Output
223 # Output
224 magic_name, _, first = first.partition(' ')
224 magic_name, _, first = first.partition(' ')
225 magic_name = magic_name.lstrip(ESC_MAGIC2)
225 magic_name = magic_name.lstrip(ESC_MAGIC2)
226 line = tpl % (magic_name, first, '\n'.join(body))
226 line = tpl % (magic_name, first, '\n'.join(body))
227
227
228 def _strip_prompts(prompt1_re, prompt2_re):
228 def _strip_prompts(prompt1_re, prompt2_re):
229 """Remove matching input prompts from a block of input."""
229 """Remove matching input prompts from a block of input."""
230 line = ''
230 line = ''
231 while True:
231 while True:
232 line = (yield line)
232 line = (yield line)
233
233
234 if line is None:
234 if line is None:
235 continue
235 continue
236
236
237 m = prompt1_re.match(line)
237 m = prompt1_re.match(line)
238 if m:
238 if m:
239 while m:
239 while m:
240 line = (yield line[len(m.group(0)):])
240 line = (yield line[len(m.group(0)):])
241 if line is None:
241 if line is None:
242 break
242 break
243 m = prompt2_re.match(line)
243 m = prompt2_re.match(line)
244 else:
244 else:
245 # Prompts not in input - wait for reset
245 # Prompts not in input - wait for reset
246 while line is not None:
246 while line is not None:
247 line = (yield line)
247 line = (yield line)
248
248
249 @CoroutineInputTransformer
249 @CoroutineInputTransformer
250 def classic_prompt():
250 def classic_prompt():
251 prompt1_re = re.compile(r'^(>>> )')
251 prompt1_re = re.compile(r'^(>>> )')
252 prompt2_re = re.compile(r'^(>>> |^\.\.\. )')
252 prompt2_re = re.compile(r'^(>>> |^\.\.\. )')
253 return _strip_prompts(prompt1_re, prompt2_re)
253 return _strip_prompts(prompt1_re, prompt2_re)
254
254
255 classic_prompt.look_in_string = True
255 classic_prompt.look_in_string = True
256
256
257 @CoroutineInputTransformer
257 @CoroutineInputTransformer
258 def ipy_prompt():
258 def ipy_prompt():
259 prompt1_re = re.compile(r'^In \[\d+\]: ')
259 prompt1_re = re.compile(r'^In \[\d+\]: ')
260 prompt2_re = re.compile(r'^(In \[\d+\]: |^\ \ \ \.\.\.+: )')
260 prompt2_re = re.compile(r'^(In \[\d+\]: |^\ \ \ \.\.\.+: )')
261 return _strip_prompts(prompt1_re, prompt2_re)
261 return _strip_prompts(prompt1_re, prompt2_re)
262
262
263 ipy_prompt.look_in_string = True
263 ipy_prompt.look_in_string = True
264
264
265 @CoroutineInputTransformer
265 @CoroutineInputTransformer
266 def leading_indent():
266 def leading_indent():
267 space_re = re.compile(r'^[ \t]+')
267 space_re = re.compile(r'^[ \t]+')
268 line = ''
268 line = ''
269 while True:
269 while True:
270 line = (yield line)
270 line = (yield line)
271
271
272 if line is None:
272 if line is None:
273 continue
273 continue
274
274
275 m = space_re.match(line)
275 m = space_re.match(line)
276 if m:
276 if m:
277 space = m.group(0)
277 space = m.group(0)
278 while line is not None:
278 while line is not None:
279 if line.startswith(space):
279 if line.startswith(space):
280 line = line[len(space):]
280 line = line[len(space):]
281 line = (yield line)
281 line = (yield line)
282 else:
282 else:
283 # No leading spaces - wait for reset
283 # No leading spaces - wait for reset
284 while line is not None:
284 while line is not None:
285 line = (yield line)
285 line = (yield line)
286
286
287 leading_indent.look_in_string = True
287 leading_indent.look_in_string = True
288
288
289 def _special_assignment(assignment_re, template):
289 def _special_assignment(assignment_re, template):
290 line = ''
290 line = ''
291 while True:
291 while True:
292 line = (yield line)
292 line = (yield line)
293 if not line or line.isspace():
293 if not line or line.isspace():
294 continue
294 continue
295
295
296 m = assignment_re.match(line)
296 m = assignment_re.match(line)
297 if not m:
297 if not m:
298 continue
298 continue
299
299
300 parts = []
300 parts = []
301 while line is not None:
301 while line is not None:
302 parts.append(line.rstrip('\\'))
302 parts.append(line.rstrip('\\'))
303 if not line.endswith('\\'):
303 if not line.endswith('\\'):
304 break
304 break
305 line = (yield None)
305 line = (yield None)
306
306
307 # Output
307 # Output
308 whole = assignment_re.match(' '.join(parts))
308 whole = assignment_re.match(' '.join(parts))
309 line = template % (whole.group('lhs'), whole.group('cmd'))
309 line = template % (whole.group('lhs'), whole.group('cmd'))
310
310
311 @CoroutineInputTransformer
311 @CoroutineInputTransformer
312 def assign_from_system():
312 def assign_from_system():
313 assignment_re = re.compile(r'(?P<lhs>(\s*)([\w\.]+)((\s*,\s*[\w\.]+)*))'
313 assignment_re = re.compile(r'(?P<lhs>(\s*)([\w\.]+)((\s*,\s*[\w\.]+)*))'
314 r'\s*=\s*!\s*(?P<cmd>.*)')
314 r'\s*=\s*!\s*(?P<cmd>.*)')
315 template = '%s = get_ipython().getoutput(%r)'
315 template = '%s = get_ipython().getoutput(%r)'
316 return _special_assignment(assignment_re, template)
316 return _special_assignment(assignment_re, template)
317
317
318 @CoroutineInputTransformer
318 @CoroutineInputTransformer
319 def assign_from_magic():
319 def assign_from_magic():
320 assignment_re = re.compile(r'(?P<lhs>(\s*)([\w\.]+)((\s*,\s*[\w\.]+)*))'
320 assignment_re = re.compile(r'(?P<lhs>(\s*)([\w\.]+)((\s*,\s*[\w\.]+)*))'
321 r'\s*=\s*%\s*(?P<cmd>.*)')
321 r'\s*=\s*%\s*(?P<cmd>.*)')
322 template = '%s = get_ipython().magic(%r)'
322 template = '%s = get_ipython().magic(%r)'
323 return _special_assignment(assignment_re, template)
323 return _special_assignment(assignment_re, template)
General Comments 0
You need to be logged in to leave comments. Login now