##// END OF EJS Templates
Final cleanups responding to Brian's code review....
Fernando Perez -
Show More
@@ -1,798 +1,817 b''
1 """Analysis of text input into executable blocks.
1 """Analysis of text input into executable blocks.
2
2
3 The main class in this module, :class:`InputSplitter`, is designed to break
3 The main class in this module, :class:`InputSplitter`, is designed to break
4 input from either interactive, line-by-line environments or block-based ones,
4 input from either interactive, line-by-line environments or block-based ones,
5 into standalone blocks that can be executed by Python as 'single' statements
5 into standalone blocks that can be executed by Python as 'single' statements
6 (thus triggering sys.displayhook).
6 (thus triggering sys.displayhook).
7
7
8 A companion, :class:`IPythonInputSplitter`, provides the same functionality but
9 with full support for the extended IPython syntax (magics, system calls, etc).
10
8 For more details, see the class docstring below.
11 For more details, see the class docstring below.
9
12
13 ToDo
14 ----
15
16 - Naming cleanups. The tr_* names aren't the most elegant, though now they are
17 at least just attributes of a class so not really very exposed.
18
19 - Think about the best way to support dynamic things: automagic, autocall,
20 macros, etc.
21
22 - Think of a better heuristic for the application of the transforms in
23 IPythonInputSplitter.push() than looking at the buffer ending in ':'. Idea:
24 track indentation change events (indent, dedent, nothing) and apply them only
25 if the indentation went up, but not otherwise.
26
27 - Think of the cleanest way for supporting user-specified transformations (the
28 user prefilters we had before).
29
10 Authors
30 Authors
31 -------
11
32
12 * Fernando Perez
33 * Fernando Perez
13 * Brian Granger
34 * Brian Granger
14 """
35 """
15 #-----------------------------------------------------------------------------
36 #-----------------------------------------------------------------------------
16 # Copyright (C) 2010 The IPython Development Team
37 # Copyright (C) 2010 The IPython Development Team
17 #
38 #
18 # Distributed under the terms of the BSD License. The full license is in
39 # Distributed under the terms of the BSD License. The full license is in
19 # the file COPYING, distributed as part of this software.
40 # the file COPYING, distributed as part of this software.
20 #-----------------------------------------------------------------------------
41 #-----------------------------------------------------------------------------
21
42
22 #-----------------------------------------------------------------------------
43 #-----------------------------------------------------------------------------
23 # Imports
44 # Imports
24 #-----------------------------------------------------------------------------
45 #-----------------------------------------------------------------------------
25 # stdlib
46 # stdlib
26 import codeop
47 import codeop
27 import re
48 import re
28 import sys
49 import sys
29
50
30 # IPython modules
51 # IPython modules
31 from IPython.utils.text import make_quoted_expr
52 from IPython.utils.text import make_quoted_expr
32
53
33 #-----------------------------------------------------------------------------
54 #-----------------------------------------------------------------------------
34 # Globals
55 # Globals
35 #-----------------------------------------------------------------------------
56 #-----------------------------------------------------------------------------
36
57
37 # The escape sequences that define the syntax transformations IPython will
58 # The escape sequences that define the syntax transformations IPython will
38 # apply to user input. These can NOT be just changed here: many regular
59 # apply to user input. These can NOT be just changed here: many regular
39 # expressions and other parts of the code may use their hardcoded values, and
60 # expressions and other parts of the code may use their hardcoded values, and
40 # for all intents and purposes they constitute the 'IPython syntax', so they
61 # for all intents and purposes they constitute the 'IPython syntax', so they
41 # should be considered fixed.
62 # should be considered fixed.
42
63
43 ESC_SHELL = '!'
64 ESC_SHELL = '!'
44 ESC_SH_CAP = '!!'
65 ESC_SH_CAP = '!!'
45 ESC_HELP = '?'
66 ESC_HELP = '?'
46 ESC_HELP2 = '??'
67 ESC_HELP2 = '??'
47 ESC_MAGIC = '%'
68 ESC_MAGIC = '%'
48 ESC_QUOTE = ','
69 ESC_QUOTE = ','
49 ESC_QUOTE2 = ';'
70 ESC_QUOTE2 = ';'
50 ESC_PAREN = '/'
71 ESC_PAREN = '/'
51
72
52 #-----------------------------------------------------------------------------
73 #-----------------------------------------------------------------------------
53 # Utilities
74 # Utilities
54 #-----------------------------------------------------------------------------
75 #-----------------------------------------------------------------------------
55
76
56 # FIXME: These are general-purpose utilities that later can be moved to the
77 # FIXME: These are general-purpose utilities that later can be moved to the
57 # general ward. Kept here for now because we're being very strict about test
78 # general ward. Kept here for now because we're being very strict about test
58 # coverage with this code, and this lets us ensure that we keep 100% coverage
79 # coverage with this code, and this lets us ensure that we keep 100% coverage
59 # while developing.
80 # while developing.
60
81
61 # compiled regexps for autoindent management
82 # compiled regexps for autoindent management
62 dedent_re = re.compile(r'^\s+raise|^\s+return|^\s+pass')
83 dedent_re = re.compile(r'^\s+raise|^\s+return|^\s+pass')
63 ini_spaces_re = re.compile(r'^([ \t\r\f\v]+)')
84 ini_spaces_re = re.compile(r'^([ \t\r\f\v]+)')
64
85
65
86
66 def num_ini_spaces(s):
87 def num_ini_spaces(s):
67 """Return the number of initial spaces in a string.
88 """Return the number of initial spaces in a string.
68
89
69 Note that tabs are counted as a single space. For now, we do *not* support
90 Note that tabs are counted as a single space. For now, we do *not* support
70 mixing of tabs and spaces in the user's input.
91 mixing of tabs and spaces in the user's input.
71
92
72 Parameters
93 Parameters
73 ----------
94 ----------
74 s : string
95 s : string
75
96
76 Returns
97 Returns
77 -------
98 -------
78 n : int
99 n : int
79 """
100 """
80
101
81 ini_spaces = ini_spaces_re.match(s)
102 ini_spaces = ini_spaces_re.match(s)
82 if ini_spaces:
103 if ini_spaces:
83 return ini_spaces.end()
104 return ini_spaces.end()
84 else:
105 else:
85 return 0
106 return 0
86
107
87
108
88 def remove_comments(src):
109 def remove_comments(src):
89 """Remove all comments from input source.
110 """Remove all comments from input source.
90
111
91 Note: comments are NOT recognized inside of strings!
112 Note: comments are NOT recognized inside of strings!
92
113
93 Parameters
114 Parameters
94 ----------
115 ----------
95 src : string
116 src : string
96 A single or multiline input string.
117 A single or multiline input string.
97
118
98 Returns
119 Returns
99 -------
120 -------
100 String with all Python comments removed.
121 String with all Python comments removed.
101 """
122 """
102
123
103 return re.sub('#.*', '', src)
124 return re.sub('#.*', '', src)
104
125
105
126
106 def get_input_encoding():
127 def get_input_encoding():
107 """Return the default standard input encoding.
128 """Return the default standard input encoding.
108
129
109 If sys.stdin has no encoding, 'ascii' is returned."""
130 If sys.stdin has no encoding, 'ascii' is returned."""
110 # There are strange environments for which sys.stdin.encoding is None. We
131 # There are strange environments for which sys.stdin.encoding is None. We
111 # ensure that a valid encoding is returned.
132 # ensure that a valid encoding is returned.
112 encoding = getattr(sys.stdin, 'encoding', None)
133 encoding = getattr(sys.stdin, 'encoding', None)
113 if encoding is None:
134 if encoding is None:
114 encoding = 'ascii'
135 encoding = 'ascii'
115 return encoding
136 return encoding
116
137
117 #-----------------------------------------------------------------------------
138 #-----------------------------------------------------------------------------
118 # Classes and functions for normal Python syntax handling
139 # Classes and functions for normal Python syntax handling
119 #-----------------------------------------------------------------------------
140 #-----------------------------------------------------------------------------
120
141
121 class InputSplitter(object):
142 class InputSplitter(object):
122 """An object that can split Python source input in executable blocks.
143 """An object that can split Python source input in executable blocks.
123
144
124 This object is designed to be used in one of two basic modes:
145 This object is designed to be used in one of two basic modes:
125
146
126 1. By feeding it python source line-by-line, using :meth:`push`. In this
147 1. By feeding it python source line-by-line, using :meth:`push`. In this
127 mode, it will return on each push whether the currently pushed code
148 mode, it will return on each push whether the currently pushed code
128 could be executed already. In addition, it provides a method called
149 could be executed already. In addition, it provides a method called
129 :meth:`push_accepts_more` that can be used to query whether more input
150 :meth:`push_accepts_more` that can be used to query whether more input
130 can be pushed into a single interactive block.
151 can be pushed into a single interactive block.
131
152
132 2. By calling :meth:`split_blocks` with a single, multiline Python string,
153 2. By calling :meth:`split_blocks` with a single, multiline Python string,
133 that is then split into blocks each of which can be executed
154 that is then split into blocks each of which can be executed
134 interactively as a single statement.
155 interactively as a single statement.
135
156
136 This is a simple example of how an interactive terminal-based client can use
157 This is a simple example of how an interactive terminal-based client can use
137 this tool::
158 this tool::
138
159
139 isp = InputSplitter()
160 isp = InputSplitter()
140 while isp.push_accepts_more():
161 while isp.push_accepts_more():
141 indent = ' '*isp.indent_spaces
162 indent = ' '*isp.indent_spaces
142 prompt = '>>> ' + indent
163 prompt = '>>> ' + indent
143 line = indent + raw_input(prompt)
164 line = indent + raw_input(prompt)
144 isp.push(line)
165 isp.push(line)
145 print 'Input source was:\n', isp.source_reset(),
166 print 'Input source was:\n', isp.source_reset(),
146 """
167 """
147 # Number of spaces of indentation computed from input that has been pushed
168 # Number of spaces of indentation computed from input that has been pushed
148 # so far. This is the attributes callers should query to get the current
169 # so far. This is the attributes callers should query to get the current
149 # indentation level, in order to provide auto-indent facilities.
170 # indentation level, in order to provide auto-indent facilities.
150 indent_spaces = 0
171 indent_spaces = 0
151 # String, indicating the default input encoding. It is computed by default
172 # String, indicating the default input encoding. It is computed by default
152 # at initialization time via get_input_encoding(), but it can be reset by a
173 # at initialization time via get_input_encoding(), but it can be reset by a
153 # client with specific knowledge of the encoding.
174 # client with specific knowledge of the encoding.
154 encoding = ''
175 encoding = ''
155 # String where the current full source input is stored, properly encoded.
176 # String where the current full source input is stored, properly encoded.
156 # Reading this attribute is the normal way of querying the currently pushed
177 # Reading this attribute is the normal way of querying the currently pushed
157 # source code, that has been properly encoded.
178 # source code, that has been properly encoded.
158 source = ''
179 source = ''
159 # Code object corresponding to the current source. It is automatically
180 # Code object corresponding to the current source. It is automatically
160 # synced to the source, so it can be queried at any time to obtain the code
181 # synced to the source, so it can be queried at any time to obtain the code
161 # object; it will be None if the source doesn't compile to valid Python.
182 # object; it will be None if the source doesn't compile to valid Python.
162 code = None
183 code = None
163 # Input mode
184 # Input mode
164 input_mode = 'append'
185 input_mode = 'append'
165
186
166 # Private attributes
187 # Private attributes
167
188
168 # List with lines of input accumulated so far
189 # List with lines of input accumulated so far
169 _buffer = None
190 _buffer = None
170 # Command compiler
191 # Command compiler
171 _compile = None
192 _compile = None
172 # Mark when input has changed indentation all the way back to flush-left
193 # Mark when input has changed indentation all the way back to flush-left
173 _full_dedent = False
194 _full_dedent = False
174 # Boolean indicating whether the current block is complete
195 # Boolean indicating whether the current block is complete
175 _is_complete = None
196 _is_complete = None
176
197
177 def __init__(self, input_mode=None):
198 def __init__(self, input_mode=None):
178 """Create a new InputSplitter instance.
199 """Create a new InputSplitter instance.
179
200
180 Parameters
201 Parameters
181 ----------
202 ----------
182 input_mode : str
203 input_mode : str
183
204
184 One of 'append', 'replace', default is 'append'. This controls how
205 One of 'append', 'replace', default is 'append'. This controls how
185 new inputs are used: in 'append' mode, they are appended to the
206 new inputs are used: in 'append' mode, they are appended to the
186 existing buffer and the whole buffer is compiled; in 'replace' mode,
207 existing buffer and the whole buffer is compiled; in 'replace' mode,
187 each new input completely replaces all prior inputs. Replace mode is
208 each new input completely replaces all prior inputs. Replace mode is
188 thus equivalent to prepending a full reset() to every push() call.
209 thus equivalent to prepending a full reset() to every push() call.
189
210
190 In practice, line-oriented clients likely want to use 'append' mode
211 In practice, line-oriented clients likely want to use 'append' mode
191 while block-oriented ones will want to use 'replace'.
212 while block-oriented ones will want to use 'replace'.
192 """
213 """
193 self._buffer = []
214 self._buffer = []
194 self._compile = codeop.CommandCompiler()
215 self._compile = codeop.CommandCompiler()
195 self.encoding = get_input_encoding()
216 self.encoding = get_input_encoding()
196 self.input_mode = InputSplitter.input_mode if input_mode is None \
217 self.input_mode = InputSplitter.input_mode if input_mode is None \
197 else input_mode
218 else input_mode
198
219
199 def reset(self):
220 def reset(self):
200 """Reset the input buffer and associated state."""
221 """Reset the input buffer and associated state."""
201 self.indent_spaces = 0
222 self.indent_spaces = 0
202 self._buffer[:] = []
223 self._buffer[:] = []
203 self.source = ''
224 self.source = ''
204 self.code = None
225 self.code = None
205 self._is_complete = False
226 self._is_complete = False
206 self._full_dedent = False
227 self._full_dedent = False
207
228
208 def source_reset(self):
229 def source_reset(self):
209 """Return the input source and perform a full reset.
230 """Return the input source and perform a full reset.
210 """
231 """
211 out = self.source
232 out = self.source
212 self.reset()
233 self.reset()
213 return out
234 return out
214
235
215 def push(self, lines):
236 def push(self, lines):
216 """Push one ore more lines of input.
237 """Push one ore more lines of input.
217
238
218 This stores the given lines and returns a status code indicating
239 This stores the given lines and returns a status code indicating
219 whether the code forms a complete Python block or not.
240 whether the code forms a complete Python block or not.
220
241
221 Any exceptions generated in compilation are swallowed, but if an
242 Any exceptions generated in compilation are swallowed, but if an
222 exception was produced, the method returns True.
243 exception was produced, the method returns True.
223
244
224 Parameters
245 Parameters
225 ----------
246 ----------
226 lines : string
247 lines : string
227 One or more lines of Python input.
248 One or more lines of Python input.
228
249
229 Returns
250 Returns
230 -------
251 -------
231 is_complete : boolean
252 is_complete : boolean
232 True if the current input source (the result of the current input
253 True if the current input source (the result of the current input
233 plus prior inputs) forms a complete Python execution block. Note that
254 plus prior inputs) forms a complete Python execution block. Note that
234 this value is also stored as a private attribute (_is_complete), so it
255 this value is also stored as a private attribute (_is_complete), so it
235 can be queried at any time.
256 can be queried at any time.
236 """
257 """
237 if self.input_mode == 'replace':
258 if self.input_mode == 'replace':
238 self.reset()
259 self.reset()
239
260
240 # If the source code has leading blanks, add 'if 1:\n' to it
261 # If the source code has leading blanks, add 'if 1:\n' to it
241 # this allows execution of indented pasted code. It is tempting
262 # this allows execution of indented pasted code. It is tempting
242 # to add '\n' at the end of source to run commands like ' a=1'
263 # to add '\n' at the end of source to run commands like ' a=1'
243 # directly, but this fails for more complicated scenarios
264 # directly, but this fails for more complicated scenarios
244 if not self._buffer and lines[:1] in [' ', '\t']:
265 if not self._buffer and lines[:1] in [' ', '\t']:
245 lines = 'if 1:\n%s' % lines
266 lines = 'if 1:\n%s' % lines
246
267
247 self._store(lines)
268 self._store(lines)
248 source = self.source
269 source = self.source
249
270
250 # Before calling _compile(), reset the code object to None so that if an
271 # Before calling _compile(), reset the code object to None so that if an
251 # exception is raised in compilation, we don't mislead by having
272 # exception is raised in compilation, we don't mislead by having
252 # inconsistent code/source attributes.
273 # inconsistent code/source attributes.
253 self.code, self._is_complete = None, None
274 self.code, self._is_complete = None, None
254
275
255 self._update_indent(lines)
276 self._update_indent(lines)
256 try:
277 try:
257 self.code = self._compile(source)
278 self.code = self._compile(source)
258 # Invalid syntax can produce any of a number of different errors from
279 # Invalid syntax can produce any of a number of different errors from
259 # inside the compiler, so we have to catch them all. Syntax errors
280 # inside the compiler, so we have to catch them all. Syntax errors
260 # immediately produce a 'ready' block, so the invalid Python can be
281 # immediately produce a 'ready' block, so the invalid Python can be
261 # sent to the kernel for evaluation with possible ipython
282 # sent to the kernel for evaluation with possible ipython
262 # special-syntax conversion.
283 # special-syntax conversion.
263 except (SyntaxError, OverflowError, ValueError, TypeError,
284 except (SyntaxError, OverflowError, ValueError, TypeError,
264 MemoryError):
285 MemoryError):
265 self._is_complete = True
286 self._is_complete = True
266 else:
287 else:
267 # Compilation didn't produce any exceptions (though it may not have
288 # Compilation didn't produce any exceptions (though it may not have
268 # given a complete code object)
289 # given a complete code object)
269 self._is_complete = self.code is not None
290 self._is_complete = self.code is not None
270
291
271 return self._is_complete
292 return self._is_complete
272
293
273 def push_accepts_more(self):
294 def push_accepts_more(self):
274 """Return whether a block of interactive input can accept more input.
295 """Return whether a block of interactive input can accept more input.
275
296
276 This method is meant to be used by line-oriented frontends, who need to
297 This method is meant to be used by line-oriented frontends, who need to
277 guess whether a block is complete or not based solely on prior and
298 guess whether a block is complete or not based solely on prior and
278 current input lines. The InputSplitter considers it has a complete
299 current input lines. The InputSplitter considers it has a complete
279 interactive block and will not accept more input only when either a
300 interactive block and will not accept more input only when either a
280 SyntaxError is raised, or *all* of the following are true:
301 SyntaxError is raised, or *all* of the following are true:
281
302
282 1. The input compiles to a complete statement.
303 1. The input compiles to a complete statement.
283
304
284 2. The indentation level is flush-left (because if we are indented,
305 2. The indentation level is flush-left (because if we are indented,
285 like inside a function definition or for loop, we need to keep
306 like inside a function definition or for loop, we need to keep
286 reading new input).
307 reading new input).
287
308
288 3. There is one extra line consisting only of whitespace.
309 3. There is one extra line consisting only of whitespace.
289
310
290 Because of condition #3, this method should be used only by
311 Because of condition #3, this method should be used only by
291 *line-oriented* frontends, since it means that intermediate blank lines
312 *line-oriented* frontends, since it means that intermediate blank lines
292 are not allowed in function definitions (or any other indented block).
313 are not allowed in function definitions (or any other indented block).
293
314
294 Block-oriented frontends that have a separate keyboard event to
315 Block-oriented frontends that have a separate keyboard event to
295 indicate execution should use the :meth:`split_blocks` method instead.
316 indicate execution should use the :meth:`split_blocks` method instead.
296
317
297 If the current input produces a syntax error, this method immediately
318 If the current input produces a syntax error, this method immediately
298 returns False but does *not* raise the syntax error exception, as
319 returns False but does *not* raise the syntax error exception, as
299 typically clients will want to send invalid syntax to an execution
320 typically clients will want to send invalid syntax to an execution
300 backend which might convert the invalid syntax into valid Python via
321 backend which might convert the invalid syntax into valid Python via
301 one of the dynamic IPython mechanisms.
322 one of the dynamic IPython mechanisms.
302 """
323 """
303
324
304 if not self._is_complete:
325 if not self._is_complete:
305 return True
326 return True
306
327
307 if self.indent_spaces==0:
328 if self.indent_spaces==0:
308 return False
329 return False
309
330
310 last_line = self.source.splitlines()[-1]
331 last_line = self.source.splitlines()[-1]
311 return bool(last_line and not last_line.isspace())
332 return bool(last_line and not last_line.isspace())
312
333
313 def split_blocks(self, lines):
334 def split_blocks(self, lines):
314 """Split a multiline string into multiple input blocks.
335 """Split a multiline string into multiple input blocks.
315
336
316 Note: this method starts by performing a full reset().
337 Note: this method starts by performing a full reset().
317
338
318 Parameters
339 Parameters
319 ----------
340 ----------
320 lines : str
341 lines : str
321 A possibly multiline string.
342 A possibly multiline string.
322
343
323 Returns
344 Returns
324 -------
345 -------
325 blocks : list
346 blocks : list
326 A list of strings, each possibly multiline. Each string corresponds
347 A list of strings, each possibly multiline. Each string corresponds
327 to a single block that can be compiled in 'single' mode (unless it
348 to a single block that can be compiled in 'single' mode (unless it
328 has a syntax error)."""
349 has a syntax error)."""
329
350
330 # This code is fairly delicate. If you make any changes here, make
351 # This code is fairly delicate. If you make any changes here, make
331 # absolutely sure that you do run the full test suite and ALL tests
352 # absolutely sure that you do run the full test suite and ALL tests
332 # pass.
353 # pass.
333
354
334 self.reset()
355 self.reset()
335 blocks = []
356 blocks = []
336
357
337 # Reversed copy so we can use pop() efficiently and consume the input
358 # Reversed copy so we can use pop() efficiently and consume the input
338 # as a stack
359 # as a stack
339 lines = lines.splitlines()[::-1]
360 lines = lines.splitlines()[::-1]
340 # Outer loop over all input
361 # Outer loop over all input
341 while lines:
362 while lines:
342 # Inner loop to build each block
363 # Inner loop to build each block
343 while True:
364 while True:
344 # Safety exit from inner loop
365 # Safety exit from inner loop
345 if not lines:
366 if not lines:
346 break
367 break
347 # Grab next line but don't push it yet
368 # Grab next line but don't push it yet
348 next_line = lines.pop()
369 next_line = lines.pop()
349 # Blank/empty lines are pushed as-is
370 # Blank/empty lines are pushed as-is
350 if not next_line or next_line.isspace():
371 if not next_line or next_line.isspace():
351 self.push(next_line)
372 self.push(next_line)
352 continue
373 continue
353
374
354 # Check indentation changes caused by the *next* line
375 # Check indentation changes caused by the *next* line
355 indent_spaces, _full_dedent = self._find_indent(next_line)
376 indent_spaces, _full_dedent = self._find_indent(next_line)
356
377
357 # If the next line causes a dedent, it can be for two differnt
378 # If the next line causes a dedent, it can be for two differnt
358 # reasons: either an explicit de-dent by the user or a
379 # reasons: either an explicit de-dent by the user or a
359 # return/raise/pass statement. These MUST be handled
380 # return/raise/pass statement. These MUST be handled
360 # separately:
381 # separately:
361 #
382 #
362 # 1. the first case is only detected when the actual explicit
383 # 1. the first case is only detected when the actual explicit
363 # dedent happens, and that would be the *first* line of a *new*
384 # dedent happens, and that would be the *first* line of a *new*
364 # block. Thus, we must put the line back into the input buffer
385 # block. Thus, we must put the line back into the input buffer
365 # so that it starts a new block on the next pass.
386 # so that it starts a new block on the next pass.
366 #
387 #
367 # 2. the second case is detected in the line before the actual
388 # 2. the second case is detected in the line before the actual
368 # dedent happens, so , we consume the line and we can break out
389 # dedent happens, so , we consume the line and we can break out
369 # to start a new block.
390 # to start a new block.
370
391
371 # Case 1, explicit dedent causes a break
392 # Case 1, explicit dedent causes a break
372 if _full_dedent and not next_line.startswith(' '):
393 if _full_dedent and not next_line.startswith(' '):
373 lines.append(next_line)
394 lines.append(next_line)
374 break
395 break
375
396
376 # Otherwise any line is pushed
397 # Otherwise any line is pushed
377 self.push(next_line)
398 self.push(next_line)
378
399
379 # Case 2, full dedent with full block ready:
400 # Case 2, full dedent with full block ready:
380 if _full_dedent or \
401 if _full_dedent or \
381 self.indent_spaces==0 and not self.push_accepts_more():
402 self.indent_spaces==0 and not self.push_accepts_more():
382 break
403 break
383 # Form the new block with the current source input
404 # Form the new block with the current source input
384 blocks.append(self.source_reset())
405 blocks.append(self.source_reset())
385
406
386 return blocks
407 return blocks
387
408
388 #------------------------------------------------------------------------
409 #------------------------------------------------------------------------
389 # Private interface
410 # Private interface
390 #------------------------------------------------------------------------
411 #------------------------------------------------------------------------
391
412
392 def _find_indent(self, line):
413 def _find_indent(self, line):
393 """Compute the new indentation level for a single line.
414 """Compute the new indentation level for a single line.
394
415
395 Parameters
416 Parameters
396 ----------
417 ----------
397 line : str
418 line : str
398 A single new line of non-whitespace, non-comment Python input.
419 A single new line of non-whitespace, non-comment Python input.
399
420
400 Returns
421 Returns
401 -------
422 -------
402 indent_spaces : int
423 indent_spaces : int
403 New value for the indent level (it may be equal to self.indent_spaces
424 New value for the indent level (it may be equal to self.indent_spaces
404 if indentation doesn't change.
425 if indentation doesn't change.
405
426
406 full_dedent : boolean
427 full_dedent : boolean
407 Whether the new line causes a full flush-left dedent.
428 Whether the new line causes a full flush-left dedent.
408 """
429 """
409 indent_spaces = self.indent_spaces
430 indent_spaces = self.indent_spaces
410 full_dedent = self._full_dedent
431 full_dedent = self._full_dedent
411
432
412 inisp = num_ini_spaces(line)
433 inisp = num_ini_spaces(line)
413 if inisp < indent_spaces:
434 if inisp < indent_spaces:
414 indent_spaces = inisp
435 indent_spaces = inisp
415 if indent_spaces <= 0:
436 if indent_spaces <= 0:
416 #print 'Full dedent in text',self.source # dbg
437 #print 'Full dedent in text',self.source # dbg
417 full_dedent = True
438 full_dedent = True
418
439
419 if line[-1] == ':':
440 if line[-1] == ':':
420 indent_spaces += 4
441 indent_spaces += 4
421 elif dedent_re.match(line):
442 elif dedent_re.match(line):
422 indent_spaces -= 4
443 indent_spaces -= 4
423 if indent_spaces <= 0:
444 if indent_spaces <= 0:
424 full_dedent = True
445 full_dedent = True
425
446
426 # Safety
447 # Safety
427 if indent_spaces < 0:
448 if indent_spaces < 0:
428 indent_spaces = 0
449 indent_spaces = 0
429 #print 'safety' # dbg
450 #print 'safety' # dbg
430
451
431 return indent_spaces, full_dedent
452 return indent_spaces, full_dedent
432
453
433 def _update_indent(self, lines):
454 def _update_indent(self, lines):
434 for line in remove_comments(lines).splitlines():
455 for line in remove_comments(lines).splitlines():
435 if line and not line.isspace():
456 if line and not line.isspace():
436 self.indent_spaces, self._full_dedent = self._find_indent(line)
457 self.indent_spaces, self._full_dedent = self._find_indent(line)
437
458
438 def _store(self, lines):
459 def _store(self, lines):
439 """Store one or more lines of input.
460 """Store one or more lines of input.
440
461
441 If input lines are not newline-terminated, a newline is automatically
462 If input lines are not newline-terminated, a newline is automatically
442 appended."""
463 appended."""
443
464
444 if lines.endswith('\n'):
465 if lines.endswith('\n'):
445 self._buffer.append(lines)
466 self._buffer.append(lines)
446 else:
467 else:
447 self._buffer.append(lines+'\n')
468 self._buffer.append(lines+'\n')
448 self._set_source()
469 self._set_source()
449
470
450 def _set_source(self):
471 def _set_source(self):
451 self.source = ''.join(self._buffer).encode(self.encoding)
472 self.source = ''.join(self._buffer).encode(self.encoding)
452
473
453
474
454 #-----------------------------------------------------------------------------
475 #-----------------------------------------------------------------------------
455 # Functions and classes for IPython-specific syntactic support
476 # Functions and classes for IPython-specific syntactic support
456 #-----------------------------------------------------------------------------
477 #-----------------------------------------------------------------------------
457
478
458 # RegExp for splitting line contents into pre-char//first word-method//rest.
479 # RegExp for splitting line contents into pre-char//first word-method//rest.
459 # For clarity, each group in on one line.
480 # For clarity, each group in on one line.
460
481
461 line_split = re.compile("""
482 line_split = re.compile("""
462 ^(\s*) # any leading space
483 ^(\s*) # any leading space
463 ([,;/%]|!!?|\?\??) # escape character or characters
484 ([,;/%]|!!?|\?\??) # escape character or characters
464 \s*([\w\.]*) # function/method part (mix of \w and '.')
485 \s*([\w\.]*) # function/method part (mix of \w and '.')
465 (\s+.*$|$) # rest of line
486 (\s+.*$|$) # rest of line
466 """, re.VERBOSE)
487 """, re.VERBOSE)
467
488
468
489
469 def split_user_input(line):
490 def split_user_input(line):
470 """Split user input into early whitespace, esc-char, function part and rest.
491 """Split user input into early whitespace, esc-char, function part and rest.
471
492
472 This is currently handles lines with '=' in them in a very inconsistent
493 This is currently handles lines with '=' in them in a very inconsistent
473 manner.
494 manner.
474
495
475 Examples
496 Examples
476 ========
497 ========
477 >>> split_user_input('x=1')
498 >>> split_user_input('x=1')
478 ('', '', 'x=1', '')
499 ('', '', 'x=1', '')
479 >>> split_user_input('?')
500 >>> split_user_input('?')
480 ('', '?', '', '')
501 ('', '?', '', '')
481 >>> split_user_input('??')
502 >>> split_user_input('??')
482 ('', '??', '', '')
503 ('', '??', '', '')
483 >>> split_user_input(' ?')
504 >>> split_user_input(' ?')
484 (' ', '?', '', '')
505 (' ', '?', '', '')
485 >>> split_user_input(' ??')
506 >>> split_user_input(' ??')
486 (' ', '??', '', '')
507 (' ', '??', '', '')
487 >>> split_user_input('??x')
508 >>> split_user_input('??x')
488 ('', '??', 'x', '')
509 ('', '??', 'x', '')
489 >>> split_user_input('?x=1')
510 >>> split_user_input('?x=1')
490 ('', '', '?x=1', '')
511 ('', '', '?x=1', '')
491 >>> split_user_input('!ls')
512 >>> split_user_input('!ls')
492 ('', '!', 'ls', '')
513 ('', '!', 'ls', '')
493 >>> split_user_input(' !ls')
514 >>> split_user_input(' !ls')
494 (' ', '!', 'ls', '')
515 (' ', '!', 'ls', '')
495 >>> split_user_input('!!ls')
516 >>> split_user_input('!!ls')
496 ('', '!!', 'ls', '')
517 ('', '!!', 'ls', '')
497 >>> split_user_input(' !!ls')
518 >>> split_user_input(' !!ls')
498 (' ', '!!', 'ls', '')
519 (' ', '!!', 'ls', '')
499 >>> split_user_input(',ls')
520 >>> split_user_input(',ls')
500 ('', ',', 'ls', '')
521 ('', ',', 'ls', '')
501 >>> split_user_input(';ls')
522 >>> split_user_input(';ls')
502 ('', ';', 'ls', '')
523 ('', ';', 'ls', '')
503 >>> split_user_input(' ;ls')
524 >>> split_user_input(' ;ls')
504 (' ', ';', 'ls', '')
525 (' ', ';', 'ls', '')
505 >>> split_user_input('f.g(x)')
526 >>> split_user_input('f.g(x)')
506 ('', '', 'f.g(x)', '')
527 ('', '', 'f.g(x)', '')
507 >>> split_user_input('f.g (x)')
528 >>> split_user_input('f.g (x)')
508 ('', '', 'f.g', '(x)')
529 ('', '', 'f.g', '(x)')
509 """
530 """
510 match = line_split.match(line)
531 match = line_split.match(line)
511 if match:
532 if match:
512 lspace, esc, fpart, rest = match.groups()
533 lspace, esc, fpart, rest = match.groups()
513 else:
534 else:
514 # print "match failed for line '%s'" % line
535 # print "match failed for line '%s'" % line
515 try:
536 try:
516 fpart, rest = line.split(None,1)
537 fpart, rest = line.split(None, 1)
517 except ValueError:
538 except ValueError:
518 # print "split failed for line '%s'" % line
539 # print "split failed for line '%s'" % line
519 fpart, rest = line,''
540 fpart, rest = line,''
520 lspace = re.match('^(\s*)(.*)',line).groups()[0]
541 lspace = re.match('^(\s*)(.*)', line).groups()[0]
521 esc = ''
542 esc = ''
522
543
523 # fpart has to be a valid python identifier, so it better be only pure
544 # fpart has to be a valid python identifier, so it better be only pure
524 # ascii, no unicode:
545 # ascii, no unicode:
525 try:
546 try:
526 fpart = fpart.encode('ascii')
547 fpart = fpart.encode('ascii')
527 except UnicodeEncodeError:
548 except UnicodeEncodeError:
528 lspace = unicode(lspace)
549 lspace = unicode(lspace)
529 rest = fpart + u' ' + rest
550 rest = fpart + u' ' + rest
530 fpart = u''
551 fpart = u''
531
552
532 #print 'line:<%s>' % line # dbg
553 #print 'line:<%s>' % line # dbg
533 #print 'esc <%s> fpart <%s> rest <%s>' % (esc,fpart.strip(),rest) # dbg
554 #print 'esc <%s> fpart <%s> rest <%s>' % (esc,fpart.strip(),rest) # dbg
534 return lspace, esc, fpart.strip(), rest.lstrip()
555 return lspace, esc, fpart.strip(), rest.lstrip()
535
556
536
557
537 # The escaped translators ALL receive a line where their own escape has been
558 # The escaped translators ALL receive a line where their own escape has been
538 # stripped. Only '?' is valid at the end of the line, all others can only be
559 # stripped. Only '?' is valid at the end of the line, all others can only be
539 # placed at the start.
560 # placed at the start.
540
561
541 class LineInfo(object):
562 class LineInfo(object):
542 """A single line of input and associated info.
563 """A single line of input and associated info.
543
564
544 This is a utility class that mostly wraps the output of
565 This is a utility class that mostly wraps the output of
545 :func:`split_user_input` into a convenient object to be passed around
566 :func:`split_user_input` into a convenient object to be passed around
546 during input transformations.
567 during input transformations.
547
568
548 Includes the following as properties:
569 Includes the following as properties:
549
570
550 line
571 line
551 The original, raw line
572 The original, raw line
552
573
553 lspace
574 lspace
554 Any early whitespace before actual text starts.
575 Any early whitespace before actual text starts.
555
576
556 esc
577 esc
557 The initial esc character (or characters, for double-char escapes like
578 The initial esc character (or characters, for double-char escapes like
558 '??' or '!!').
579 '??' or '!!').
559
580
560 pre_char
561 The escape character(s) in esc or the empty string if there isn't one.
562
563 fpart
581 fpart
564 The 'function part', which is basically the maximal initial sequence
582 The 'function part', which is basically the maximal initial sequence
565 of valid python identifiers and the '.' character. This is what is
583 of valid python identifiers and the '.' character. This is what is
566 checked for alias and magic transformations, used for auto-calling,
584 checked for alias and magic transformations, used for auto-calling,
567 etc.
585 etc.
568
586
569 rest
587 rest
570 Everything else on the line.
588 Everything else on the line.
571 """
589 """
572 def __init__(self, line):
590 def __init__(self, line):
573 self.line = line
591 self.line = line
574 self.lspace, self.esc, self.fpart, self.rest = \
592 self.lspace, self.esc, self.fpart, self.rest = \
575 split_user_input(line)
593 split_user_input(line)
576
594
577 def __str__(self):
595 def __str__(self):
578 return "LineInfo [%s|%s|%s|%s]" % (self.lspace, self.esc,
596 return "LineInfo [%s|%s|%s|%s]" % (self.lspace, self.esc,
579 self.fpart, self.rest)
597 self.fpart, self.rest)
580
598
581
599
582 # Transformations of the special syntaxes that don't rely on an explicit escape
600 # Transformations of the special syntaxes that don't rely on an explicit escape
583 # character but instead on patterns on the input line
601 # character but instead on patterns on the input line
584
602
585 # The core transformations are implemented as standalone functions that can be
603 # The core transformations are implemented as standalone functions that can be
586 # tested and validated in isolation. Each of these uses a regexp, we
604 # tested and validated in isolation. Each of these uses a regexp, we
587 # pre-compile these and keep them close to each function definition for clarity
605 # pre-compile these and keep them close to each function definition for clarity
588
606
589 _assign_system_re = re.compile(r'(?P<lhs>(\s*)([\w\.]+)((\s*,\s*[\w\.]+)*))'
607 _assign_system_re = re.compile(r'(?P<lhs>(\s*)([\w\.]+)((\s*,\s*[\w\.]+)*))'
590 r'\s*=\s*!\s*(?P<cmd>.*)')
608 r'\s*=\s*!\s*(?P<cmd>.*)')
591
609
592 def transform_assign_system(line):
610 def transform_assign_system(line):
593 """Handle the `files = !ls` syntax."""
611 """Handle the `files = !ls` syntax."""
594 # FIXME: This transforms the line to use %sc, but we've listed that magic
612 # FIXME: This transforms the line to use %sc, but we've listed that magic
595 # as deprecated. We should then implement this functionality in a
613 # as deprecated. We should then implement this functionality in a
596 # standalone api that we can transform to, without going through a
614 # standalone api that we can transform to, without going through a
597 # deprecated magic.
615 # deprecated magic.
598 m = _assign_system_re.match(line)
616 m = _assign_system_re.match(line)
599 if m is not None:
617 if m is not None:
600 cmd = m.group('cmd')
618 cmd = m.group('cmd')
601 lhs = m.group('lhs')
619 lhs = m.group('lhs')
602 expr = make_quoted_expr("sc -l = %s" % cmd)
620 expr = make_quoted_expr("sc -l = %s" % cmd)
603 new_line = '%s = get_ipython().magic(%s)' % (lhs, expr)
621 new_line = '%s = get_ipython().magic(%s)' % (lhs, expr)
604 return new_line
622 return new_line
605 return line
623 return line
606
624
607
625
608 _assign_magic_re = re.compile(r'(?P<lhs>(\s*)([\w\.]+)((\s*,\s*[\w\.]+)*))'
626 _assign_magic_re = re.compile(r'(?P<lhs>(\s*)([\w\.]+)((\s*,\s*[\w\.]+)*))'
609 r'\s*=\s*%\s*(?P<cmd>.*)')
627 r'\s*=\s*%\s*(?P<cmd>.*)')
610
628
611 def transform_assign_magic(line):
629 def transform_assign_magic(line):
612 """Handle the `a = %who` syntax."""
630 """Handle the `a = %who` syntax."""
613 m = _assign_magic_re.match(line)
631 m = _assign_magic_re.match(line)
614 if m is not None:
632 if m is not None:
615 cmd = m.group('cmd')
633 cmd = m.group('cmd')
616 lhs = m.group('lhs')
634 lhs = m.group('lhs')
617 expr = make_quoted_expr(cmd)
635 expr = make_quoted_expr(cmd)
618 new_line = '%s = get_ipython().magic(%s)' % (lhs, expr)
636 new_line = '%s = get_ipython().magic(%s)' % (lhs, expr)
619 return new_line
637 return new_line
620 return line
638 return line
621
639
622
640
623 _classic_prompt_re = re.compile(r'^([ \t]*>>> |^[ \t]*\.\.\. )')
641 _classic_prompt_re = re.compile(r'^([ \t]*>>> |^[ \t]*\.\.\. )')
624
642
625 def transform_classic_prompt(line):
643 def transform_classic_prompt(line):
626 """Handle inputs that start with '>>> ' syntax."""
644 """Handle inputs that start with '>>> ' syntax."""
627
645
628 if not line or line.isspace():
646 if not line or line.isspace():
629 return line
647 return line
630 m = _classic_prompt_re.match(line)
648 m = _classic_prompt_re.match(line)
631 if m:
649 if m:
632 return line[len(m.group(0)):]
650 return line[len(m.group(0)):]
633 else:
651 else:
634 return line
652 return line
635
653
636
654
637 _ipy_prompt_re = re.compile(r'^([ \t]*In \[\d+\]: |^[ \t]*\ \ \ \.\.\.+: )')
655 _ipy_prompt_re = re.compile(r'^([ \t]*In \[\d+\]: |^[ \t]*\ \ \ \.\.\.+: )')
638
656
639 def transform_ipy_prompt(line):
657 def transform_ipy_prompt(line):
640 """Handle inputs that start classic IPython prompt syntax."""
658 """Handle inputs that start classic IPython prompt syntax."""
641
659
642 if not line or line.isspace():
660 if not line or line.isspace():
643 return line
661 return line
644 m = _ipy_prompt_re.match(line)
662 m = _ipy_prompt_re.match(line)
645 if m:
663 if m:
646 return line[len(m.group(0)):]
664 return line[len(m.group(0)):]
647 else:
665 else:
648 return line
666 return line
649
667
650
668
651 def transform_unescaped(line):
669 class EscapedTransformer(object):
652 """Transform lines that are explicitly escaped out.
670 """Class to transform lines that are explicitly escaped out."""
653
671
654 This calls to the above transform_* functions for the actual line
672 def __init__(self):
655 translations.
673 tr = { ESC_SHELL : self.tr_system,
656
674 ESC_SH_CAP : self.tr_system2,
657 Parameters
675 ESC_HELP : self.tr_help,
658 ----------
676 ESC_HELP2 : self.tr_help,
659 line : str
677 ESC_MAGIC : self.tr_magic,
660 A single line of input to be transformed.
678 ESC_QUOTE : self.tr_quote,
661
679 ESC_QUOTE2 : self.tr_quote2,
662 Returns
680 ESC_PAREN : self.tr_paren }
663 -------
681 self.tr = tr
664 new_line : str
665 Transformed line, which may be identical to the original."""
666
667 if not line or line.isspace():
668 return line
669
670 new_line = line
671 for f in [transform_assign_system, transform_assign_magic,
672 transform_classic_prompt, transform_ipy_prompt ] :
673 new_line = f(new_line)
674 return new_line
675
682
676 # Support for syntax transformations that use explicit escapes typed by the
683 # Support for syntax transformations that use explicit escapes typed by the
677 # user at the beginning of a line
684 # user at the beginning of a line
678
685 @staticmethod
679 def tr_system(line_info):
686 def tr_system(line_info):
680 "Translate lines escaped with: !"
687 "Translate lines escaped with: !"
681 cmd = line_info.line.lstrip().lstrip(ESC_SHELL)
688 cmd = line_info.line.lstrip().lstrip(ESC_SHELL)
682 return '%sget_ipython().system(%s)' % (line_info.lspace,
689 return '%sget_ipython().system(%s)' % (line_info.lspace,
683 make_quoted_expr(cmd))
690 make_quoted_expr(cmd))
684
691
685
692 @staticmethod
686 def tr_system2(line_info):
693 def tr_system2(line_info):
687 "Translate lines escaped with: !!"
694 "Translate lines escaped with: !!"
688 cmd = line_info.line.lstrip()[2:]
695 cmd = line_info.line.lstrip()[2:]
689 return '%sget_ipython().getoutput(%s)' % (line_info.lspace,
696 return '%sget_ipython().getoutput(%s)' % (line_info.lspace,
690 make_quoted_expr(cmd))
697 make_quoted_expr(cmd))
691
698
692
699 @staticmethod
693 def tr_help(line_info):
700 def tr_help(line_info):
694 "Translate lines escaped with: ?/??"
701 "Translate lines escaped with: ?/??"
695 # A naked help line should just fire the intro help screen
702 # A naked help line should just fire the intro help screen
696 if not line_info.line[1:]:
703 if not line_info.line[1:]:
697 return 'get_ipython().show_usage()'
704 return 'get_ipython().show_usage()'
698
705
699 # There may be one or two '?' at the end, move them to the front so that
706 # There may be one or two '?' at the end, move them to the front so that
700 # the rest of the logic can assume escapes are at the start
707 # the rest of the logic can assume escapes are at the start
701 line = line_info.line
708 line = line_info.line
702 if line.endswith('?'):
709 if line.endswith('?'):
703 line = line[-1] + line[:-1]
710 line = line[-1] + line[:-1]
704 if line.endswith('?'):
711 if line.endswith('?'):
705 line = line[-1] + line[:-1]
712 line = line[-1] + line[:-1]
706 line_info = LineInfo(line)
713 line_info = LineInfo(line)
707
714
708 # From here on, simply choose which level of detail to get.
715 # From here on, simply choose which level of detail to get.
709 if line_info.esc == '?':
716 if line_info.esc == '?':
710 pinfo = 'pinfo'
717 pinfo = 'pinfo'
711 elif line_info.esc == '??':
718 elif line_info.esc == '??':
712 pinfo = 'pinfo2'
719 pinfo = 'pinfo2'
713
720
714 tpl = '%sget_ipython().magic("%s %s")'
721 tpl = '%sget_ipython().magic("%s %s")'
715 return tpl % (line_info.lspace, pinfo,
722 return tpl % (line_info.lspace, pinfo,
716 ' '.join([line_info.fpart, line_info.rest]).strip())
723 ' '.join([line_info.fpart, line_info.rest]).strip())
717
724
718
725 @staticmethod
719 def tr_magic(line_info):
726 def tr_magic(line_info):
720 "Translate lines escaped with: %"
727 "Translate lines escaped with: %"
721 tpl = '%sget_ipython().magic(%s)'
728 tpl = '%sget_ipython().magic(%s)'
722 cmd = make_quoted_expr(' '.join([line_info.fpart,
729 cmd = make_quoted_expr(' '.join([line_info.fpart,
723 line_info.rest])).strip()
730 line_info.rest])).strip()
724 return tpl % (line_info.lspace, cmd)
731 return tpl % (line_info.lspace, cmd)
725
732
726
733 @staticmethod
727 def tr_quote(line_info):
734 def tr_quote(line_info):
728 "Translate lines escaped with: ,"
735 "Translate lines escaped with: ,"
729 return '%s%s("%s")' % (line_info.lspace, line_info.fpart,
736 return '%s%s("%s")' % (line_info.lspace, line_info.fpart,
730 '", "'.join(line_info.rest.split()) )
737 '", "'.join(line_info.rest.split()) )
731
738
732
739 @staticmethod
733 def tr_quote2(line_info):
740 def tr_quote2(line_info):
734 "Translate lines escaped with: ;"
741 "Translate lines escaped with: ;"
735 return '%s%s("%s")' % (line_info.lspace, line_info.fpart,
742 return '%s%s("%s")' % (line_info.lspace, line_info.fpart,
736 line_info.rest)
743 line_info.rest)
737
744
738
745 @staticmethod
739 def tr_paren(line_info):
746 def tr_paren(line_info):
740 "Translate lines escaped with: /"
747 "Translate lines escaped with: /"
741 return '%s%s(%s)' % (line_info.lspace, line_info.fpart,
748 return '%s%s(%s)' % (line_info.lspace, line_info.fpart,
742 ", ".join(line_info.rest.split()))
749 ", ".join(line_info.rest.split()))
743
750
751 def __call__(self, line):
752 """Class to transform lines that are explicitly escaped out.
744
753
745 def transform_escaped(line):
754 This calls the above tr_* static methods for the actual line
746 """Transform lines that are explicitly escaped out.
755 translations."""
747
748 This calls to the above tr_* functions for the actual line translations."""
749
750 tr = { ESC_SHELL : tr_system,
751 ESC_SH_CAP : tr_system2,
752 ESC_HELP : tr_help,
753 ESC_HELP2 : tr_help,
754 ESC_MAGIC : tr_magic,
755 ESC_QUOTE : tr_quote,
756 ESC_QUOTE2 : tr_quote2,
757 ESC_PAREN : tr_paren }
758
756
759 # Empty lines just get returned unmodified
757 # Empty lines just get returned unmodified
760 if not line or line.isspace():
758 if not line or line.isspace():
761 return line
759 return line
762
760
763 # Get line endpoints, where the escapes can be
761 # Get line endpoints, where the escapes can be
764 line_info = LineInfo(line)
762 line_info = LineInfo(line)
765
763
766 # If the escape is not at the start, only '?' needs to be special-cased.
764 # If the escape is not at the start, only '?' needs to be special-cased.
767 # All other escapes are only valid at the start
765 # All other escapes are only valid at the start
768 if not line_info.esc in tr:
766 if not line_info.esc in self.tr:
769 if line.endswith(ESC_HELP):
767 if line.endswith(ESC_HELP):
770 return tr_help(line_info)
768 return self.tr_help(line_info)
771 else:
769 else:
772 # If we don't recognize the escape, don't modify the line
770 # If we don't recognize the escape, don't modify the line
773 return line
771 return line
774
772
775 return tr[line_info.esc](line_info)
773 return self.tr[line_info.esc](line_info)
774
775 # A function-looking object to be used by the rest of the code. The purpose of
776 # the class in this case is to organize related functionality, more than to
777 # manage state.
778 transform_escaped = EscapedTransformer()
776
779
777
780
778 class IPythonInputSplitter(InputSplitter):
781 class IPythonInputSplitter(InputSplitter):
779 """An input splitter that recognizes all of IPython's special syntax."""
782 """An input splitter that recognizes all of IPython's special syntax."""
780
783
781 def push(self, lines):
784 def push(self, lines):
782 """Push one or more lines of IPython input.
785 """Push one or more lines of IPython input.
783 """
786 """
787 if not lines:
788 return super(IPythonInputSplitter, self).push(lines)
789
790 lines_list = lines.splitlines()
791
792 transforms = [transform_escaped, transform_assign_system,
793 transform_assign_magic, transform_ipy_prompt,
794 transform_classic_prompt]
795
796 # Transform logic
797 #
784 # We only apply the line transformers to the input if we have either no
798 # We only apply the line transformers to the input if we have either no
785 # input yet, or complete input. This prevents the accidental
799 # input yet, or complete input, or if the last line of the buffer ends
800 # with ':' (opening an indented block). This prevents the accidental
786 # transformation of escapes inside multiline expressions like
801 # transformation of escapes inside multiline expressions like
787 # triple-quoted strings or parenthesized expressions.
802 # triple-quoted strings or parenthesized expressions.
788 lines_list = lines.splitlines()
803 #
789 if self._is_complete or not self._buffer:
804 # The last heuristic, while ugly, ensures that the first line of an
805 # indented block is correctly transformed.
806 #
807 # FIXME: try to find a cleaner approach for this last bit.
790
808
791 new_list = map(transform_escaped, lines_list)
809 for line in lines_list:
792 else:
810 if self._is_complete or not self._buffer or \
793 new_list = lines_list
811 (self._buffer and self._buffer[-1].rstrip().endswith(':')):
812 for f in transforms:
813 line = f(line)
794
814
795 # Now apply the unescaped transformations to each input line
815 out = super(IPythonInputSplitter, self).push(line)
796 new_list = map(transform_unescaped, new_list)
816
797 newlines = '\n'.join(new_list)
817 return out
798 return super(IPythonInputSplitter, self).push(newlines)
@@ -1,623 +1,621 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2 """Tests for the inputsplitter module.
2 """Tests for the inputsplitter module.
3 """
3 """
4 #-----------------------------------------------------------------------------
4 #-----------------------------------------------------------------------------
5 # Copyright (C) 2010 The IPython Development Team
5 # Copyright (C) 2010 The IPython Development Team
6 #
6 #
7 # Distributed under the terms of the BSD License. The full license is in
7 # Distributed under the terms of the BSD License. The full license is in
8 # the file COPYING, distributed as part of this software.
8 # the file COPYING, distributed as part of this software.
9 #-----------------------------------------------------------------------------
9 #-----------------------------------------------------------------------------
10
10
11 #-----------------------------------------------------------------------------
11 #-----------------------------------------------------------------------------
12 # Imports
12 # Imports
13 #-----------------------------------------------------------------------------
13 #-----------------------------------------------------------------------------
14 # stdlib
14 # stdlib
15 import unittest
15 import unittest
16 import sys
16 import sys
17
17
18 # Third party
18 # Third party
19 import nose.tools as nt
19 import nose.tools as nt
20
20
21 # Our own
21 # Our own
22 from IPython.core import inputsplitter as isp
22 from IPython.core import inputsplitter as isp
23
23
24 #-----------------------------------------------------------------------------
24 #-----------------------------------------------------------------------------
25 # Semi-complete examples (also used as tests)
25 # Semi-complete examples (also used as tests)
26 #-----------------------------------------------------------------------------
26 #-----------------------------------------------------------------------------
27
27
28 # Note: at the bottom, there's a slightly more complete version of this that
28 # Note: at the bottom, there's a slightly more complete version of this that
29 # can be useful during development of code here.
29 # can be useful during development of code here.
30
30
31 def mini_interactive_loop(raw_input):
31 def mini_interactive_loop(raw_input):
32 """Minimal example of the logic of an interactive interpreter loop.
32 """Minimal example of the logic of an interactive interpreter loop.
33
33
34 This serves as an example, and it is used by the test system with a fake
34 This serves as an example, and it is used by the test system with a fake
35 raw_input that simulates interactive input."""
35 raw_input that simulates interactive input."""
36
36
37 from IPython.core.inputsplitter import InputSplitter
37 from IPython.core.inputsplitter import InputSplitter
38
38
39 isp = InputSplitter()
39 isp = InputSplitter()
40 # In practice, this input loop would be wrapped in an outside loop to read
40 # In practice, this input loop would be wrapped in an outside loop to read
41 # input indefinitely, until some exit/quit command was issued. Here we
41 # input indefinitely, until some exit/quit command was issued. Here we
42 # only illustrate the basic inner loop.
42 # only illustrate the basic inner loop.
43 while isp.push_accepts_more():
43 while isp.push_accepts_more():
44 indent = ' '*isp.indent_spaces
44 indent = ' '*isp.indent_spaces
45 prompt = '>>> ' + indent
45 prompt = '>>> ' + indent
46 line = indent + raw_input(prompt)
46 line = indent + raw_input(prompt)
47 isp.push(line)
47 isp.push(line)
48
48
49 # Here we just return input so we can use it in a test suite, but a real
49 # Here we just return input so we can use it in a test suite, but a real
50 # interpreter would instead send it for execution somewhere.
50 # interpreter would instead send it for execution somewhere.
51 src = isp.source_reset()
51 src = isp.source_reset()
52 #print 'Input source was:\n', src # dbg
52 #print 'Input source was:\n', src # dbg
53 return src
53 return src
54
54
55 #-----------------------------------------------------------------------------
55 #-----------------------------------------------------------------------------
56 # Test utilities, just for local use
56 # Test utilities, just for local use
57 #-----------------------------------------------------------------------------
57 #-----------------------------------------------------------------------------
58
58
59 def assemble(block):
59 def assemble(block):
60 """Assemble a block into multi-line sub-blocks."""
60 """Assemble a block into multi-line sub-blocks."""
61 return ['\n'.join(sub_block)+'\n' for sub_block in block]
61 return ['\n'.join(sub_block)+'\n' for sub_block in block]
62
62
63
63
64 def pseudo_input(lines):
64 def pseudo_input(lines):
65 """Return a function that acts like raw_input but feeds the input list."""
65 """Return a function that acts like raw_input but feeds the input list."""
66 ilines = iter(lines)
66 ilines = iter(lines)
67 def raw_in(prompt):
67 def raw_in(prompt):
68 try:
68 try:
69 return next(ilines)
69 return next(ilines)
70 except StopIteration:
70 except StopIteration:
71 return ''
71 return ''
72 return raw_in
72 return raw_in
73
73
74 #-----------------------------------------------------------------------------
74 #-----------------------------------------------------------------------------
75 # Tests
75 # Tests
76 #-----------------------------------------------------------------------------
76 #-----------------------------------------------------------------------------
77 def test_spaces():
77 def test_spaces():
78 tests = [('', 0),
78 tests = [('', 0),
79 (' ', 1),
79 (' ', 1),
80 ('\n', 0),
80 ('\n', 0),
81 (' \n', 1),
81 (' \n', 1),
82 ('x', 0),
82 ('x', 0),
83 (' x', 1),
83 (' x', 1),
84 (' x',2),
84 (' x',2),
85 (' x',4),
85 (' x',4),
86 # Note: tabs are counted as a single whitespace!
86 # Note: tabs are counted as a single whitespace!
87 ('\tx', 1),
87 ('\tx', 1),
88 ('\t x', 2),
88 ('\t x', 2),
89 ]
89 ]
90
90
91 for s, nsp in tests:
91 for s, nsp in tests:
92 nt.assert_equal(isp.num_ini_spaces(s), nsp)
92 nt.assert_equal(isp.num_ini_spaces(s), nsp)
93
93
94
94
95 def test_remove_comments():
95 def test_remove_comments():
96 tests = [('text', 'text'),
96 tests = [('text', 'text'),
97 ('text # comment', 'text '),
97 ('text # comment', 'text '),
98 ('text # comment\n', 'text \n'),
98 ('text # comment\n', 'text \n'),
99 ('text # comment \n', 'text \n'),
99 ('text # comment \n', 'text \n'),
100 ('line # c \nline\n','line \nline\n'),
100 ('line # c \nline\n','line \nline\n'),
101 ('line # c \nline#c2 \nline\nline #c\n\n',
101 ('line # c \nline#c2 \nline\nline #c\n\n',
102 'line \nline\nline\nline \n\n'),
102 'line \nline\nline\nline \n\n'),
103 ]
103 ]
104
104
105 for inp, out in tests:
105 for inp, out in tests:
106 nt.assert_equal(isp.remove_comments(inp), out)
106 nt.assert_equal(isp.remove_comments(inp), out)
107
107
108
108
109 def test_get_input_encoding():
109 def test_get_input_encoding():
110 encoding = isp.get_input_encoding()
110 encoding = isp.get_input_encoding()
111 nt.assert_true(isinstance(encoding, basestring))
111 nt.assert_true(isinstance(encoding, basestring))
112 # simple-minded check that at least encoding a simple string works with the
112 # simple-minded check that at least encoding a simple string works with the
113 # encoding we got.
113 # encoding we got.
114 nt.assert_equal('test'.encode(encoding), 'test')
114 nt.assert_equal('test'.encode(encoding), 'test')
115
115
116
116
117 class NoInputEncodingTestCase(unittest.TestCase):
117 class NoInputEncodingTestCase(unittest.TestCase):
118 def setUp(self):
118 def setUp(self):
119 self.old_stdin = sys.stdin
119 self.old_stdin = sys.stdin
120 class X: pass
120 class X: pass
121 fake_stdin = X()
121 fake_stdin = X()
122 sys.stdin = fake_stdin
122 sys.stdin = fake_stdin
123
123
124 def test(self):
124 def test(self):
125 # Verify that if sys.stdin has no 'encoding' attribute we do the right
125 # Verify that if sys.stdin has no 'encoding' attribute we do the right
126 # thing
126 # thing
127 enc = isp.get_input_encoding()
127 enc = isp.get_input_encoding()
128 self.assertEqual(enc, 'ascii')
128 self.assertEqual(enc, 'ascii')
129
129
130 def tearDown(self):
130 def tearDown(self):
131 sys.stdin = self.old_stdin
131 sys.stdin = self.old_stdin
132
132
133
133
134 class InputSplitterTestCase(unittest.TestCase):
134 class InputSplitterTestCase(unittest.TestCase):
135 def setUp(self):
135 def setUp(self):
136 self.isp = isp.InputSplitter()
136 self.isp = isp.InputSplitter()
137
137
138 def test_reset(self):
138 def test_reset(self):
139 isp = self.isp
139 isp = self.isp
140 isp.push('x=1')
140 isp.push('x=1')
141 isp.reset()
141 isp.reset()
142 self.assertEqual(isp._buffer, [])
142 self.assertEqual(isp._buffer, [])
143 self.assertEqual(isp.indent_spaces, 0)
143 self.assertEqual(isp.indent_spaces, 0)
144 self.assertEqual(isp.source, '')
144 self.assertEqual(isp.source, '')
145 self.assertEqual(isp.code, None)
145 self.assertEqual(isp.code, None)
146 self.assertEqual(isp._is_complete, False)
146 self.assertEqual(isp._is_complete, False)
147
147
148 def test_source(self):
148 def test_source(self):
149 self.isp._store('1')
149 self.isp._store('1')
150 self.isp._store('2')
150 self.isp._store('2')
151 self.assertEqual(self.isp.source, '1\n2\n')
151 self.assertEqual(self.isp.source, '1\n2\n')
152 self.assertTrue(len(self.isp._buffer)>0)
152 self.assertTrue(len(self.isp._buffer)>0)
153 self.assertEqual(self.isp.source_reset(), '1\n2\n')
153 self.assertEqual(self.isp.source_reset(), '1\n2\n')
154 self.assertEqual(self.isp._buffer, [])
154 self.assertEqual(self.isp._buffer, [])
155 self.assertEqual(self.isp.source, '')
155 self.assertEqual(self.isp.source, '')
156
156
157 def test_indent(self):
157 def test_indent(self):
158 isp = self.isp # shorthand
158 isp = self.isp # shorthand
159 isp.push('x=1')
159 isp.push('x=1')
160 self.assertEqual(isp.indent_spaces, 0)
160 self.assertEqual(isp.indent_spaces, 0)
161 isp.push('if 1:\n x=1')
161 isp.push('if 1:\n x=1')
162 self.assertEqual(isp.indent_spaces, 4)
162 self.assertEqual(isp.indent_spaces, 4)
163 isp.push('y=2\n')
163 isp.push('y=2\n')
164 self.assertEqual(isp.indent_spaces, 0)
164 self.assertEqual(isp.indent_spaces, 0)
165 isp.push('if 1:')
165 isp.push('if 1:')
166 self.assertEqual(isp.indent_spaces, 4)
166 self.assertEqual(isp.indent_spaces, 4)
167 isp.push(' x=1')
167 isp.push(' x=1')
168 self.assertEqual(isp.indent_spaces, 4)
168 self.assertEqual(isp.indent_spaces, 4)
169 # Blank lines shouldn't change the indent level
169 # Blank lines shouldn't change the indent level
170 isp.push(' '*2)
170 isp.push(' '*2)
171 self.assertEqual(isp.indent_spaces, 4)
171 self.assertEqual(isp.indent_spaces, 4)
172
172
173 def test_indent2(self):
173 def test_indent2(self):
174 isp = self.isp
174 isp = self.isp
175 # When a multiline statement contains parens or multiline strings, we
175 # When a multiline statement contains parens or multiline strings, we
176 # shouldn't get confused.
176 # shouldn't get confused.
177 isp.push("if 1:")
177 isp.push("if 1:")
178 isp.push(" x = (1+\n 2)")
178 isp.push(" x = (1+\n 2)")
179 self.assertEqual(isp.indent_spaces, 4)
179 self.assertEqual(isp.indent_spaces, 4)
180
180
181 def test_dedent(self):
181 def test_dedent(self):
182 isp = self.isp # shorthand
182 isp = self.isp # shorthand
183 isp.push('if 1:')
183 isp.push('if 1:')
184 self.assertEqual(isp.indent_spaces, 4)
184 self.assertEqual(isp.indent_spaces, 4)
185 isp.push(' pass')
185 isp.push(' pass')
186 self.assertEqual(isp.indent_spaces, 0)
186 self.assertEqual(isp.indent_spaces, 0)
187
187
188 def test_push(self):
188 def test_push(self):
189 isp = self.isp
189 isp = self.isp
190 self.assertTrue(isp.push('x=1'))
190 self.assertTrue(isp.push('x=1'))
191
191
192 def test_push2(self):
192 def test_push2(self):
193 isp = self.isp
193 isp = self.isp
194 self.assertFalse(isp.push('if 1:'))
194 self.assertFalse(isp.push('if 1:'))
195 for line in [' x=1', '# a comment', ' y=2']:
195 for line in [' x=1', '# a comment', ' y=2']:
196 self.assertTrue(isp.push(line))
196 self.assertTrue(isp.push(line))
197
197
198 def test_push3(self):
198 def test_push3(self):
199 """Test input with leading whitespace"""
199 """Test input with leading whitespace"""
200 isp = self.isp
200 isp = self.isp
201 isp.push(' x=1')
201 isp.push(' x=1')
202 isp.push(' y=2')
202 isp.push(' y=2')
203 self.assertEqual(isp.source, 'if 1:\n x=1\n y=2\n')
203 self.assertEqual(isp.source, 'if 1:\n x=1\n y=2\n')
204
204
205 def test_replace_mode(self):
205 def test_replace_mode(self):
206 isp = self.isp
206 isp = self.isp
207 isp.input_mode = 'replace'
207 isp.input_mode = 'replace'
208 isp.push('x=1')
208 isp.push('x=1')
209 self.assertEqual(isp.source, 'x=1\n')
209 self.assertEqual(isp.source, 'x=1\n')
210 isp.push('x=2')
210 isp.push('x=2')
211 self.assertEqual(isp.source, 'x=2\n')
211 self.assertEqual(isp.source, 'x=2\n')
212
212
213 def test_push_accepts_more(self):
213 def test_push_accepts_more(self):
214 isp = self.isp
214 isp = self.isp
215 isp.push('x=1')
215 isp.push('x=1')
216 self.assertFalse(isp.push_accepts_more())
216 self.assertFalse(isp.push_accepts_more())
217
217
218 def test_push_accepts_more2(self):
218 def test_push_accepts_more2(self):
219 isp = self.isp
219 isp = self.isp
220 isp.push('if 1:')
220 isp.push('if 1:')
221 self.assertTrue(isp.push_accepts_more())
221 self.assertTrue(isp.push_accepts_more())
222 isp.push(' x=1')
222 isp.push(' x=1')
223 self.assertTrue(isp.push_accepts_more())
223 self.assertTrue(isp.push_accepts_more())
224 isp.push('')
224 isp.push('')
225 self.assertFalse(isp.push_accepts_more())
225 self.assertFalse(isp.push_accepts_more())
226
226
227 def test_push_accepts_more3(self):
227 def test_push_accepts_more3(self):
228 isp = self.isp
228 isp = self.isp
229 isp.push("x = (2+\n3)")
229 isp.push("x = (2+\n3)")
230 self.assertFalse(isp.push_accepts_more())
230 self.assertFalse(isp.push_accepts_more())
231
231
232 def test_push_accepts_more4(self):
232 def test_push_accepts_more4(self):
233 isp = self.isp
233 isp = self.isp
234 # When a multiline statement contains parens or multiline strings, we
234 # When a multiline statement contains parens or multiline strings, we
235 # shouldn't get confused.
235 # shouldn't get confused.
236 # FIXME: we should be able to better handle de-dents in statements like
236 # FIXME: we should be able to better handle de-dents in statements like
237 # multiline strings and multiline expressions (continued with \ or
237 # multiline strings and multiline expressions (continued with \ or
238 # parens). Right now we aren't handling the indentation tracking quite
238 # parens). Right now we aren't handling the indentation tracking quite
239 # correctly with this, though in practice it may not be too much of a
239 # correctly with this, though in practice it may not be too much of a
240 # problem. We'll need to see.
240 # problem. We'll need to see.
241 isp.push("if 1:")
241 isp.push("if 1:")
242 isp.push(" x = (2+")
242 isp.push(" x = (2+")
243 isp.push(" 3)")
243 isp.push(" 3)")
244 self.assertTrue(isp.push_accepts_more())
244 self.assertTrue(isp.push_accepts_more())
245 isp.push(" y = 3")
245 isp.push(" y = 3")
246 self.assertTrue(isp.push_accepts_more())
246 self.assertTrue(isp.push_accepts_more())
247 isp.push('')
247 isp.push('')
248 self.assertFalse(isp.push_accepts_more())
248 self.assertFalse(isp.push_accepts_more())
249
249
250 def test_syntax_error(self):
250 def test_syntax_error(self):
251 isp = self.isp
251 isp = self.isp
252 # Syntax errors immediately produce a 'ready' block, so the invalid
252 # Syntax errors immediately produce a 'ready' block, so the invalid
253 # Python can be sent to the kernel for evaluation with possible ipython
253 # Python can be sent to the kernel for evaluation with possible ipython
254 # special-syntax conversion.
254 # special-syntax conversion.
255 isp.push('run foo')
255 isp.push('run foo')
256 self.assertFalse(isp.push_accepts_more())
256 self.assertFalse(isp.push_accepts_more())
257
257
258 def check_split(self, block_lines, compile=True):
258 def check_split(self, block_lines, compile=True):
259 blocks = assemble(block_lines)
259 blocks = assemble(block_lines)
260 lines = ''.join(blocks)
260 lines = ''.join(blocks)
261 oblock = self.isp.split_blocks(lines)
261 oblock = self.isp.split_blocks(lines)
262 self.assertEqual(oblock, blocks)
262 self.assertEqual(oblock, blocks)
263 if compile:
263 if compile:
264 for block in blocks:
264 for block in blocks:
265 self.isp._compile(block)
265 self.isp._compile(block)
266
266
267 def test_split(self):
267 def test_split(self):
268 # All blocks of input we want to test in a list. The format for each
268 # All blocks of input we want to test in a list. The format for each
269 # block is a list of lists, with each inner lists consisting of all the
269 # block is a list of lists, with each inner lists consisting of all the
270 # lines (as single-lines) that should make up a sub-block.
270 # lines (as single-lines) that should make up a sub-block.
271
271
272 # Note: do NOT put here sub-blocks that don't compile, as the
272 # Note: do NOT put here sub-blocks that don't compile, as the
273 # check_split() routine makes a final verification pass to check that
273 # check_split() routine makes a final verification pass to check that
274 # each sub_block, as returned by split_blocks(), does compile
274 # each sub_block, as returned by split_blocks(), does compile
275 # correctly.
275 # correctly.
276 all_blocks = [ [['x=1']],
276 all_blocks = [ [['x=1']],
277
277
278 [['x=1'],
278 [['x=1'],
279 ['y=2']],
279 ['y=2']],
280
280
281 [['x=1'],
281 [['x=1'],
282 ['# a comment'],
282 ['# a comment'],
283 ['y=11']],
283 ['y=11']],
284
284
285 [['if 1:',
285 [['if 1:',
286 ' x=1'],
286 ' x=1'],
287 ['y=3']],
287 ['y=3']],
288
288
289 [['def f(x):',
289 [['def f(x):',
290 ' return x'],
290 ' return x'],
291 ['x=1']],
291 ['x=1']],
292
292
293 [['def f(x):',
293 [['def f(x):',
294 ' x+=1',
294 ' x+=1',
295 ' ',
295 ' ',
296 ' return x'],
296 ' return x'],
297 ['x=1']],
297 ['x=1']],
298
298
299 [['def f(x):',
299 [['def f(x):',
300 ' if x>0:',
300 ' if x>0:',
301 ' y=1',
301 ' y=1',
302 ' # a comment',
302 ' # a comment',
303 ' else:',
303 ' else:',
304 ' y=4',
304 ' y=4',
305 ' ',
305 ' ',
306 ' return y'],
306 ' return y'],
307 ['x=1'],
307 ['x=1'],
308 ['if 1:',
308 ['if 1:',
309 ' y=11'] ],
309 ' y=11'] ],
310
310
311 [['for i in range(10):'
311 [['for i in range(10):'
312 ' x=i**2']],
312 ' x=i**2']],
313
313
314 [['for i in range(10):'
314 [['for i in range(10):'
315 ' x=i**2'],
315 ' x=i**2'],
316 ['z = 1']],
316 ['z = 1']],
317 ]
317 ]
318 for block_lines in all_blocks:
318 for block_lines in all_blocks:
319 self.check_split(block_lines)
319 self.check_split(block_lines)
320
320
321 def test_split_syntax_errors(self):
321 def test_split_syntax_errors(self):
322 # Block splitting with invalid syntax
322 # Block splitting with invalid syntax
323 all_blocks = [ [['a syntax error']],
323 all_blocks = [ [['a syntax error']],
324
324
325 [['x=1'],
325 [['x=1'],
326 ['a syntax error']],
326 ['a syntax error']],
327
327
328 [['for i in range(10):'
328 [['for i in range(10):'
329 ' an error']],
329 ' an error']],
330
330
331 ]
331 ]
332 for block_lines in all_blocks:
332 for block_lines in all_blocks:
333 self.check_split(block_lines, compile=False)
333 self.check_split(block_lines, compile=False)
334
334
335
335
336 class InteractiveLoopTestCase(unittest.TestCase):
336 class InteractiveLoopTestCase(unittest.TestCase):
337 """Tests for an interactive loop like a python shell.
337 """Tests for an interactive loop like a python shell.
338 """
338 """
339 def check_ns(self, lines, ns):
339 def check_ns(self, lines, ns):
340 """Validate that the given input lines produce the resulting namespace.
340 """Validate that the given input lines produce the resulting namespace.
341
341
342 Note: the input lines are given exactly as they would be typed in an
342 Note: the input lines are given exactly as they would be typed in an
343 auto-indenting environment, as mini_interactive_loop above already does
343 auto-indenting environment, as mini_interactive_loop above already does
344 auto-indenting and prepends spaces to the input.
344 auto-indenting and prepends spaces to the input.
345 """
345 """
346 src = mini_interactive_loop(pseudo_input(lines))
346 src = mini_interactive_loop(pseudo_input(lines))
347 test_ns = {}
347 test_ns = {}
348 exec src in test_ns
348 exec src in test_ns
349 # We can't check that the provided ns is identical to the test_ns,
349 # We can't check that the provided ns is identical to the test_ns,
350 # because Python fills test_ns with extra keys (copyright, etc). But
350 # because Python fills test_ns with extra keys (copyright, etc). But
351 # we can check that the given dict is *contained* in test_ns
351 # we can check that the given dict is *contained* in test_ns
352 for k,v in ns.items():
352 for k,v in ns.items():
353 self.assertEqual(test_ns[k], v)
353 self.assertEqual(test_ns[k], v)
354
354
355 def test_simple(self):
355 def test_simple(self):
356 self.check_ns(['x=1'], dict(x=1))
356 self.check_ns(['x=1'], dict(x=1))
357
357
358 def test_simple2(self):
358 def test_simple2(self):
359 self.check_ns(['if 1:', 'x=2'], dict(x=2))
359 self.check_ns(['if 1:', 'x=2'], dict(x=2))
360
360
361 def test_xy(self):
361 def test_xy(self):
362 self.check_ns(['x=1; y=2'], dict(x=1, y=2))
362 self.check_ns(['x=1; y=2'], dict(x=1, y=2))
363
363
364 def test_abc(self):
364 def test_abc(self):
365 self.check_ns(['if 1:','a=1','b=2','c=3'], dict(a=1, b=2, c=3))
365 self.check_ns(['if 1:','a=1','b=2','c=3'], dict(a=1, b=2, c=3))
366
366
367 def test_multi(self):
367 def test_multi(self):
368 self.check_ns(['x =(1+','1+','2)'], dict(x=4))
368 self.check_ns(['x =(1+','1+','2)'], dict(x=4))
369
369
370
370
371 def test_LineInfo():
371 def test_LineInfo():
372 """Simple test for LineInfo construction and str()"""
372 """Simple test for LineInfo construction and str()"""
373 linfo = isp.LineInfo(' %cd /home')
373 linfo = isp.LineInfo(' %cd /home')
374 nt.assert_equals(str(linfo), 'LineInfo [ |%|cd|/home]')
374 nt.assert_equals(str(linfo), 'LineInfo [ |%|cd|/home]')
375
375
376
376
377 def test_split_user_input():
377 def test_split_user_input():
378 """Unicode test - split_user_input already has good doctests"""
378 """Unicode test - split_user_input already has good doctests"""
379 line = u"PΓ©rez Fernando"
379 line = u"PΓ©rez Fernando"
380 parts = isp.split_user_input(line)
380 parts = isp.split_user_input(line)
381 parts_expected = (u'', u'', u'', line)
381 parts_expected = (u'', u'', u'', line)
382 nt.assert_equal(parts, parts_expected)
382 nt.assert_equal(parts, parts_expected)
383
383
384
384
385 # Transformer tests
385 # Transformer tests
386 def transform_checker(tests, func):
386 def transform_checker(tests, func):
387 """Utility to loop over test inputs"""
387 """Utility to loop over test inputs"""
388 for inp, tr in tests:
388 for inp, tr in tests:
389 nt.assert_equals(func(inp), tr)
389 nt.assert_equals(func(inp), tr)
390
390
391 # Data for all the syntax tests in the form of lists of pairs of
391 # Data for all the syntax tests in the form of lists of pairs of
392 # raw/transformed input. We store it here as a global dict so that we can use
392 # raw/transformed input. We store it here as a global dict so that we can use
393 # it both within single-function tests and also to validate the behavior of the
393 # it both within single-function tests and also to validate the behavior of the
394 # larger objects
394 # larger objects
395
395
396 syntax = \
396 syntax = \
397 dict(assign_system =
397 dict(assign_system =
398 [('a =! ls', 'a = get_ipython().magic("sc -l = ls")'),
398 [('a =! ls', 'a = get_ipython().magic("sc -l = ls")'),
399 ('b = !ls', 'b = get_ipython().magic("sc -l = ls")'),
399 ('b = !ls', 'b = get_ipython().magic("sc -l = ls")'),
400 ('x=1', 'x=1'), # normal input is unmodified
400 ('x=1', 'x=1'), # normal input is unmodified
401 (' ',' '), # blank lines are kept intact
401 (' ',' '), # blank lines are kept intact
402 ],
402 ],
403
403
404 assign_magic =
404 assign_magic =
405 [('a =% who', 'a = get_ipython().magic("who")'),
405 [('a =% who', 'a = get_ipython().magic("who")'),
406 ('b = %who', 'b = get_ipython().magic("who")'),
406 ('b = %who', 'b = get_ipython().magic("who")'),
407 ('x=1', 'x=1'), # normal input is unmodified
407 ('x=1', 'x=1'), # normal input is unmodified
408 (' ',' '), # blank lines are kept intact
408 (' ',' '), # blank lines are kept intact
409 ],
409 ],
410
410
411 classic_prompt =
411 classic_prompt =
412 [('>>> x=1', 'x=1'),
412 [('>>> x=1', 'x=1'),
413 ('x=1', 'x=1'), # normal input is unmodified
413 ('x=1', 'x=1'), # normal input is unmodified
414 (' ',' '), # blank lines are kept intact
414 (' ',' '), # blank lines are kept intact
415 ],
415 ],
416
416
417 ipy_prompt =
417 ipy_prompt =
418 [('In [1]: x=1', 'x=1'),
418 [('In [1]: x=1', 'x=1'),
419 ('x=1', 'x=1'), # normal input is unmodified
419 ('x=1', 'x=1'), # normal input is unmodified
420 (' ',' '), # blank lines are kept intact
420 (' ',' '), # blank lines are kept intact
421 ],
421 ],
422
422
423 # Tests for the escape transformer to leave normal code alone
423 # Tests for the escape transformer to leave normal code alone
424 escaped_noesc =
424 escaped_noesc =
425 [ (' ', ' '),
425 [ (' ', ' '),
426 ('x=1', 'x=1'),
426 ('x=1', 'x=1'),
427 ],
427 ],
428
428
429 # System calls
429 # System calls
430 escaped_shell =
430 escaped_shell =
431 [ ('!ls', 'get_ipython().system("ls")'),
431 [ ('!ls', 'get_ipython().system("ls")'),
432 # Double-escape shell, this means to capture the output of the
432 # Double-escape shell, this means to capture the output of the
433 # subprocess and return it
433 # subprocess and return it
434 ('!!ls', 'get_ipython().getoutput("ls")'),
434 ('!!ls', 'get_ipython().getoutput("ls")'),
435 ],
435 ],
436
436
437 # Help/object info
437 # Help/object info
438 escaped_help =
438 escaped_help =
439 [ ('?', 'get_ipython().show_usage()'),
439 [ ('?', 'get_ipython().show_usage()'),
440 ('?x1', 'get_ipython().magic("pinfo x1")'),
440 ('?x1', 'get_ipython().magic("pinfo x1")'),
441 ('??x2', 'get_ipython().magic("pinfo2 x2")'),
441 ('??x2', 'get_ipython().magic("pinfo2 x2")'),
442 ('x3?', 'get_ipython().magic("pinfo x3")'),
442 ('x3?', 'get_ipython().magic("pinfo x3")'),
443 ('x4??', 'get_ipython().magic("pinfo2 x4")'),
443 ('x4??', 'get_ipython().magic("pinfo2 x4")'),
444 ],
444 ],
445
445
446 # Explicit magic calls
446 # Explicit magic calls
447 escaped_magic =
447 escaped_magic =
448 [ ('%cd', 'get_ipython().magic("cd")'),
448 [ ('%cd', 'get_ipython().magic("cd")'),
449 ('%cd /home', 'get_ipython().magic("cd /home")'),
449 ('%cd /home', 'get_ipython().magic("cd /home")'),
450 (' %magic', ' get_ipython().magic("magic")'),
450 (' %magic', ' get_ipython().magic("magic")'),
451 ],
451 ],
452
452
453 # Quoting with separate arguments
453 # Quoting with separate arguments
454 escaped_quote =
454 escaped_quote =
455 [ (',f', 'f("")'),
455 [ (',f', 'f("")'),
456 (',f x', 'f("x")'),
456 (',f x', 'f("x")'),
457 (' ,f y', ' f("y")'),
457 (' ,f y', ' f("y")'),
458 (',f a b', 'f("a", "b")'),
458 (',f a b', 'f("a", "b")'),
459 ],
459 ],
460
460
461 # Quoting with single argument
461 # Quoting with single argument
462 escaped_quote2 =
462 escaped_quote2 =
463 [ (';f', 'f("")'),
463 [ (';f', 'f("")'),
464 (';f x', 'f("x")'),
464 (';f x', 'f("x")'),
465 (' ;f y', ' f("y")'),
465 (' ;f y', ' f("y")'),
466 (';f a b', 'f("a b")'),
466 (';f a b', 'f("a b")'),
467 ],
467 ],
468
468
469 # Simply apply parens
469 # Simply apply parens
470 escaped_paren =
470 escaped_paren =
471 [ ('/f', 'f()'),
471 [ ('/f', 'f()'),
472 ('/f x', 'f(x)'),
472 ('/f x', 'f(x)'),
473 (' /f y', ' f(y)'),
473 (' /f y', ' f(y)'),
474 ('/f a b', 'f(a, b)'),
474 ('/f a b', 'f(a, b)'),
475 ],
475 ],
476
476
477 # More complex multiline tests
477 # More complex multiline tests
478 ## escaped_multiline =
478 ## escaped_multiline =
479 ## [()],
479 ## [()],
480 )
480 )
481
481
482 # multiline syntax examples. Each of these should be a list of lists, with
482 # multiline syntax examples. Each of these should be a list of lists, with
483 # each entry itself having pairs of raw/transformed input. The union (with
483 # each entry itself having pairs of raw/transformed input. The union (with
484 # '\n'.join() of the transformed inputs is what the splitter should produce
484 # '\n'.join() of the transformed inputs is what the splitter should produce
485 # when fed the raw lines one at a time via push.
485 # when fed the raw lines one at a time via push.
486 syntax_ml = \
486 syntax_ml = \
487 dict(classic_prompt =
487 dict(classic_prompt =
488 [ [('>>> for i in range(10):','for i in range(10):'),
488 [ [('>>> for i in range(10):','for i in range(10):'),
489 ('... print i',' print i'),
489 ('... print i',' print i'),
490 ('... ', ''),
490 ('... ', ''),
491 ],
491 ],
492 ],
492 ],
493
493
494 ipy_prompt =
494 ipy_prompt =
495 [ [('In [24]: for i in range(10):','for i in range(10):'),
495 [ [('In [24]: for i in range(10):','for i in range(10):'),
496 (' ....: print i',' print i'),
496 (' ....: print i',' print i'),
497 (' ....: ', ''),
497 (' ....: ', ''),
498 ],
498 ],
499 ],
499 ],
500 )
500 )
501
501
502
502
503 def test_assign_system():
503 def test_assign_system():
504 transform_checker(syntax['assign_system'], isp.transform_assign_system)
504 transform_checker(syntax['assign_system'], isp.transform_assign_system)
505
505
506
506
507 def test_assign_magic():
507 def test_assign_magic():
508 transform_checker(syntax['assign_magic'], isp.transform_assign_magic)
508 transform_checker(syntax['assign_magic'], isp.transform_assign_magic)
509
509
510
510
511 def test_classic_prompt():
511 def test_classic_prompt():
512 transform_checker(syntax['classic_prompt'], isp.transform_classic_prompt)
512 transform_checker(syntax['classic_prompt'], isp.transform_classic_prompt)
513 for example in syntax_ml['classic_prompt']:
513 for example in syntax_ml['classic_prompt']:
514 transform_checker(example, isp.transform_classic_prompt)
514 transform_checker(example, isp.transform_classic_prompt)
515
515
516
516
517 def test_ipy_prompt():
517 def test_ipy_prompt():
518 transform_checker(syntax['ipy_prompt'], isp.transform_ipy_prompt)
518 transform_checker(syntax['ipy_prompt'], isp.transform_ipy_prompt)
519 for example in syntax_ml['ipy_prompt']:
519 for example in syntax_ml['ipy_prompt']:
520 transform_checker(example, isp.transform_ipy_prompt)
520 transform_checker(example, isp.transform_ipy_prompt)
521
521
522
522
523 def test_escaped_noesc():
523 def test_escaped_noesc():
524 transform_checker(syntax['escaped_noesc'], isp.transform_escaped)
524 transform_checker(syntax['escaped_noesc'], isp.transform_escaped)
525
525
526
526
527 def test_escaped_shell():
527 def test_escaped_shell():
528 transform_checker(syntax['escaped_shell'], isp.transform_escaped)
528 transform_checker(syntax['escaped_shell'], isp.transform_escaped)
529
529
530
530
531 def test_escaped_help():
531 def test_escaped_help():
532 transform_checker(syntax['escaped_help'], isp.transform_escaped)
532 transform_checker(syntax['escaped_help'], isp.transform_escaped)
533
533
534
534
535 def test_escaped_magic():
535 def test_escaped_magic():
536 transform_checker(syntax['escaped_magic'], isp.transform_escaped)
536 transform_checker(syntax['escaped_magic'], isp.transform_escaped)
537
537
538
538
539 def test_escaped_quote():
539 def test_escaped_quote():
540 transform_checker(syntax['escaped_quote'], isp.transform_escaped)
540 transform_checker(syntax['escaped_quote'], isp.transform_escaped)
541
541
542
542
543 def test_escaped_quote2():
543 def test_escaped_quote2():
544 transform_checker(syntax['escaped_quote2'], isp.transform_escaped)
544 transform_checker(syntax['escaped_quote2'], isp.transform_escaped)
545
545
546
546
547 def test_escaped_paren():
547 def test_escaped_paren():
548 transform_checker(syntax['escaped_paren'], isp.transform_escaped)
548 transform_checker(syntax['escaped_paren'], isp.transform_escaped)
549
549
550
550
551 class IPythonInputTestCase(InputSplitterTestCase):
551 class IPythonInputTestCase(InputSplitterTestCase):
552 """By just creating a new class whose .isp is a different instance, we
552 """By just creating a new class whose .isp is a different instance, we
553 re-run the same test battery on the new input splitter.
553 re-run the same test battery on the new input splitter.
554
554
555 In addition, this runs the tests over the syntax and syntax_ml dicts that
555 In addition, this runs the tests over the syntax and syntax_ml dicts that
556 were tested by individual functions, as part of the OO interface.
556 were tested by individual functions, as part of the OO interface.
557 """
557 """
558 def setUp(self):
558 def setUp(self):
559 self.isp = isp.IPythonInputSplitter()
559 self.isp = isp.IPythonInputSplitter()
560
560
561 def test_syntax(self):
561 def test_syntax(self):
562 """Call all single-line syntax tests from the main object"""
562 """Call all single-line syntax tests from the main object"""
563 isp = self.isp
563 isp = self.isp
564 for example in syntax.itervalues():
564 for example in syntax.itervalues():
565 for raw, out_t in example:
565 for raw, out_t in example:
566 if raw.startswith(' '):
566 if raw.startswith(' '):
567 continue
567 continue
568
568
569 isp.push(raw)
569 isp.push(raw)
570 out = isp.source_reset().rstrip()
570 out = isp.source_reset().rstrip()
571 self.assertEqual(out, out_t)
571 self.assertEqual(out, out_t)
572
572
573 def test_syntax_multiline(self):
573 def test_syntax_multiline(self):
574 isp = self.isp
574 isp = self.isp
575 for example in syntax_ml.itervalues():
575 for example in syntax_ml.itervalues():
576 out_t_parts = []
576 out_t_parts = []
577 for line_pairs in example:
577 for line_pairs in example:
578 for raw, out_t_part in line_pairs:
578 for raw, out_t_part in line_pairs:
579 isp.push(raw)
579 isp.push(raw)
580 out_t_parts.append(out_t_part)
580 out_t_parts.append(out_t_part)
581
581
582 out = isp.source_reset().rstrip()
582 out = isp.source_reset().rstrip()
583 out_t = '\n'.join(out_t_parts).rstrip()
583 out_t = '\n'.join(out_t_parts).rstrip()
584 self.assertEqual(out, out_t)
584 self.assertEqual(out, out_t)
585
585
586
586
587 #-----------------------------------------------------------------------------
587 #-----------------------------------------------------------------------------
588 # Main - use as a script
588 # Main - use as a script
589 #-----------------------------------------------------------------------------
589 #-----------------------------------------------------------------------------
590
590
591 if __name__ == '__main__':
591 if __name__ == '__main__':
592 # A simple demo for interactive experimentation. This code will not get
592 # A simple demo for interactive experimentation. This code will not get
593 # picked up by any test suite. Useful mostly for illustration and during
593 # picked up by any test suite. Useful mostly for illustration and during
594 # development.
594 # development.
595 from IPython.core.inputsplitter import InputSplitter, IPythonInputSplitter
595 from IPython.core.inputsplitter import InputSplitter, IPythonInputSplitter
596
596
597 # configure here the syntax to use, prompt and whether to autoindent
597 #isp, start_prompt = InputSplitter(), '>>> '
598 #isp, start_prompt = InputSplitter(), '>>> '
598 isp, start_prompt = IPythonInputSplitter(), 'In> '
599 isp, start_prompt = IPythonInputSplitter(), 'In> '
599
600
600 autoindent = True
601 autoindent = True
601 #autoindent = False
602 #autoindent = False
602
603
603 # In practice, this input loop would be wrapped in an outside loop to read
604 # input indefinitely, until some exit/quit command was issued. Here we
605 # only illustrate the basic inner loop.
606 try:
604 try:
607 while True:
605 while True:
608 prompt = start_prompt
606 prompt = start_prompt
609 while isp.push_accepts_more():
607 while isp.push_accepts_more():
610 indent = ' '*isp.indent_spaces
608 indent = ' '*isp.indent_spaces
611 if autoindent:
609 if autoindent:
612 line = indent + raw_input(prompt+indent)
610 line = indent + raw_input(prompt+indent)
613 else:
611 else:
614 line = raw_input(prompt)
612 line = raw_input(prompt)
615 isp.push(line)
613 isp.push(line)
616 prompt = '... '
614 prompt = '... '
617
615
618 # Here we just return input so we can use it in a test suite, but a
616 # Here we just return input so we can use it in a test suite, but a
619 # real interpreter would instead send it for execution somewhere.
617 # real interpreter would instead send it for execution somewhere.
620 src = isp.source_reset()
618 src = isp.source_reset()
621 print 'Input source was:\n', src # dbg
619 print 'Input source was:\n', src
622 except EOFError:
620 except EOFError:
623 print 'Bye'
621 print 'Bye'
General Comments 0
You need to be logged in to leave comments. Login now