##// END OF EJS Templates
Fix InputTransformer to strip Qt console continuation prompts....
Thomas Kluyver -
Show More
@@ -1,535 +1,534
1 import abc
1 import abc
2 import functools
2 import functools
3 import re
3 import re
4
4
5 from IPython.core.splitinput import LineInfo
5 from IPython.core.splitinput import LineInfo
6 from IPython.utils import tokenize2
6 from IPython.utils import tokenize2
7 from IPython.utils.openpy import cookie_comment_re
7 from IPython.utils.openpy import cookie_comment_re
8 from IPython.utils.py3compat import with_metaclass, PY3
8 from IPython.utils.py3compat import with_metaclass, PY3
9 from IPython.utils.tokenize2 import generate_tokens, untokenize, TokenError
9 from IPython.utils.tokenize2 import generate_tokens, untokenize, TokenError
10
10
11 if PY3:
11 if PY3:
12 from io import StringIO
12 from io import StringIO
13 else:
13 else:
14 from StringIO import StringIO
14 from StringIO import StringIO
15
15
16 #-----------------------------------------------------------------------------
16 #-----------------------------------------------------------------------------
17 # Globals
17 # Globals
18 #-----------------------------------------------------------------------------
18 #-----------------------------------------------------------------------------
19
19
20 # The escape sequences that define the syntax transformations IPython will
20 # The escape sequences that define the syntax transformations IPython will
21 # apply to user input. These can NOT be just changed here: many regular
21 # apply to user input. These can NOT be just changed here: many regular
22 # expressions and other parts of the code may use their hardcoded values, and
22 # expressions and other parts of the code may use their hardcoded values, and
23 # for all intents and purposes they constitute the 'IPython syntax', so they
23 # for all intents and purposes they constitute the 'IPython syntax', so they
24 # should be considered fixed.
24 # should be considered fixed.
25
25
26 ESC_SHELL = '!' # Send line to underlying system shell
26 ESC_SHELL = '!' # Send line to underlying system shell
27 ESC_SH_CAP = '!!' # Send line to system shell and capture output
27 ESC_SH_CAP = '!!' # Send line to system shell and capture output
28 ESC_HELP = '?' # Find information about object
28 ESC_HELP = '?' # Find information about object
29 ESC_HELP2 = '??' # Find extra-detailed information about object
29 ESC_HELP2 = '??' # Find extra-detailed information about object
30 ESC_MAGIC = '%' # Call magic function
30 ESC_MAGIC = '%' # Call magic function
31 ESC_MAGIC2 = '%%' # Call cell-magic function
31 ESC_MAGIC2 = '%%' # Call cell-magic function
32 ESC_QUOTE = ',' # Split args on whitespace, quote each as string and call
32 ESC_QUOTE = ',' # Split args on whitespace, quote each as string and call
33 ESC_QUOTE2 = ';' # Quote all args as a single string, call
33 ESC_QUOTE2 = ';' # Quote all args as a single string, call
34 ESC_PAREN = '/' # Call first argument with rest of line as arguments
34 ESC_PAREN = '/' # Call first argument with rest of line as arguments
35
35
36 ESC_SEQUENCES = [ESC_SHELL, ESC_SH_CAP, ESC_HELP ,\
36 ESC_SEQUENCES = [ESC_SHELL, ESC_SH_CAP, ESC_HELP ,\
37 ESC_HELP2, ESC_MAGIC, ESC_MAGIC2,\
37 ESC_HELP2, ESC_MAGIC, ESC_MAGIC2,\
38 ESC_QUOTE, ESC_QUOTE2, ESC_PAREN ]
38 ESC_QUOTE, ESC_QUOTE2, ESC_PAREN ]
39
39
40
40
41 class InputTransformer(with_metaclass(abc.ABCMeta, object)):
41 class InputTransformer(with_metaclass(abc.ABCMeta, object)):
42 """Abstract base class for line-based input transformers."""
42 """Abstract base class for line-based input transformers."""
43
43
44 @abc.abstractmethod
44 @abc.abstractmethod
45 def push(self, line):
45 def push(self, line):
46 """Send a line of input to the transformer, returning the transformed
46 """Send a line of input to the transformer, returning the transformed
47 input or None if the transformer is waiting for more input.
47 input or None if the transformer is waiting for more input.
48
48
49 Must be overridden by subclasses.
49 Must be overridden by subclasses.
50 """
50 """
51 pass
51 pass
52
52
53 @abc.abstractmethod
53 @abc.abstractmethod
54 def reset(self):
54 def reset(self):
55 """Return, transformed any lines that the transformer has accumulated,
55 """Return, transformed any lines that the transformer has accumulated,
56 and reset its internal state.
56 and reset its internal state.
57
57
58 Must be overridden by subclasses.
58 Must be overridden by subclasses.
59 """
59 """
60 pass
60 pass
61
61
62 @classmethod
62 @classmethod
63 def wrap(cls, func):
63 def wrap(cls, func):
64 """Can be used by subclasses as a decorator, to return a factory that
64 """Can be used by subclasses as a decorator, to return a factory that
65 will allow instantiation with the decorated object.
65 will allow instantiation with the decorated object.
66 """
66 """
67 @functools.wraps(func)
67 @functools.wraps(func)
68 def transformer_factory(**kwargs):
68 def transformer_factory(**kwargs):
69 return cls(func, **kwargs)
69 return cls(func, **kwargs)
70
70
71 return transformer_factory
71 return transformer_factory
72
72
73 class StatelessInputTransformer(InputTransformer):
73 class StatelessInputTransformer(InputTransformer):
74 """Wrapper for a stateless input transformer implemented as a function."""
74 """Wrapper for a stateless input transformer implemented as a function."""
75 def __init__(self, func):
75 def __init__(self, func):
76 self.func = func
76 self.func = func
77
77
78 def __repr__(self):
78 def __repr__(self):
79 return "StatelessInputTransformer(func={0!r})".format(self.func)
79 return "StatelessInputTransformer(func={0!r})".format(self.func)
80
80
81 def push(self, line):
81 def push(self, line):
82 """Send a line of input to the transformer, returning the
82 """Send a line of input to the transformer, returning the
83 transformed input."""
83 transformed input."""
84 return self.func(line)
84 return self.func(line)
85
85
86 def reset(self):
86 def reset(self):
87 """No-op - exists for compatibility."""
87 """No-op - exists for compatibility."""
88 pass
88 pass
89
89
90 class CoroutineInputTransformer(InputTransformer):
90 class CoroutineInputTransformer(InputTransformer):
91 """Wrapper for an input transformer implemented as a coroutine."""
91 """Wrapper for an input transformer implemented as a coroutine."""
92 def __init__(self, coro, **kwargs):
92 def __init__(self, coro, **kwargs):
93 # Prime it
93 # Prime it
94 self.coro = coro(**kwargs)
94 self.coro = coro(**kwargs)
95 next(self.coro)
95 next(self.coro)
96
96
97 def __repr__(self):
97 def __repr__(self):
98 return "CoroutineInputTransformer(coro={0!r})".format(self.coro)
98 return "CoroutineInputTransformer(coro={0!r})".format(self.coro)
99
99
100 def push(self, line):
100 def push(self, line):
101 """Send a line of input to the transformer, returning the
101 """Send a line of input to the transformer, returning the
102 transformed input or None if the transformer is waiting for more
102 transformed input or None if the transformer is waiting for more
103 input.
103 input.
104 """
104 """
105 return self.coro.send(line)
105 return self.coro.send(line)
106
106
107 def reset(self):
107 def reset(self):
108 """Return, transformed any lines that the transformer has
108 """Return, transformed any lines that the transformer has
109 accumulated, and reset its internal state.
109 accumulated, and reset its internal state.
110 """
110 """
111 return self.coro.send(None)
111 return self.coro.send(None)
112
112
113 class TokenInputTransformer(InputTransformer):
113 class TokenInputTransformer(InputTransformer):
114 """Wrapper for a token-based input transformer.
114 """Wrapper for a token-based input transformer.
115
115
116 func should accept a list of tokens (5-tuples, see tokenize docs), and
116 func should accept a list of tokens (5-tuples, see tokenize docs), and
117 return an iterable which can be passed to tokenize.untokenize().
117 return an iterable which can be passed to tokenize.untokenize().
118 """
118 """
119 def __init__(self, func):
119 def __init__(self, func):
120 self.func = func
120 self.func = func
121 self.current_line = ""
121 self.current_line = ""
122 self.line_used = False
122 self.line_used = False
123 self.reset_tokenizer()
123 self.reset_tokenizer()
124
124
125 def reset_tokenizer(self):
125 def reset_tokenizer(self):
126 self.tokenizer = generate_tokens(self.get_line)
126 self.tokenizer = generate_tokens(self.get_line)
127
127
128 def get_line(self):
128 def get_line(self):
129 if self.line_used:
129 if self.line_used:
130 raise TokenError
130 raise TokenError
131 self.line_used = True
131 self.line_used = True
132 return self.current_line
132 return self.current_line
133
133
134 def push(self, line):
134 def push(self, line):
135 self.current_line += line + "\n"
135 self.current_line += line + "\n"
136 if self.current_line.isspace():
136 if self.current_line.isspace():
137 return self.reset()
137 return self.reset()
138
138
139 self.line_used = False
139 self.line_used = False
140 tokens = []
140 tokens = []
141 stop_at_NL = False
141 stop_at_NL = False
142 try:
142 try:
143 for intok in self.tokenizer:
143 for intok in self.tokenizer:
144 tokens.append(intok)
144 tokens.append(intok)
145 t = intok[0]
145 t = intok[0]
146 if t == tokenize2.NEWLINE or (stop_at_NL and t == tokenize2.NL):
146 if t == tokenize2.NEWLINE or (stop_at_NL and t == tokenize2.NL):
147 # Stop before we try to pull a line we don't have yet
147 # Stop before we try to pull a line we don't have yet
148 break
148 break
149 elif t == tokenize2.ERRORTOKEN:
149 elif t == tokenize2.ERRORTOKEN:
150 stop_at_NL = True
150 stop_at_NL = True
151 except TokenError:
151 except TokenError:
152 # Multi-line statement - stop and try again with the next line
152 # Multi-line statement - stop and try again with the next line
153 self.reset_tokenizer()
153 self.reset_tokenizer()
154 return None
154 return None
155
155
156 return self.output(tokens)
156 return self.output(tokens)
157
157
158 def output(self, tokens):
158 def output(self, tokens):
159 self.current_line = ""
159 self.current_line = ""
160 self.reset_tokenizer()
160 self.reset_tokenizer()
161 return untokenize(self.func(tokens)).rstrip('\n')
161 return untokenize(self.func(tokens)).rstrip('\n')
162
162
163 def reset(self):
163 def reset(self):
164 l = self.current_line
164 l = self.current_line
165 self.current_line = ""
165 self.current_line = ""
166 self.reset_tokenizer()
166 self.reset_tokenizer()
167 if l:
167 if l:
168 return l.rstrip('\n')
168 return l.rstrip('\n')
169
169
170 class assemble_python_lines(TokenInputTransformer):
170 class assemble_python_lines(TokenInputTransformer):
171 def __init__(self):
171 def __init__(self):
172 super(assemble_python_lines, self).__init__(None)
172 super(assemble_python_lines, self).__init__(None)
173
173
174 def output(self, tokens):
174 def output(self, tokens):
175 return self.reset()
175 return self.reset()
176
176
177 @CoroutineInputTransformer.wrap
177 @CoroutineInputTransformer.wrap
178 def assemble_logical_lines():
178 def assemble_logical_lines():
179 """Join lines following explicit line continuations (\)"""
179 """Join lines following explicit line continuations (\)"""
180 line = ''
180 line = ''
181 while True:
181 while True:
182 line = (yield line)
182 line = (yield line)
183 if not line or line.isspace():
183 if not line or line.isspace():
184 continue
184 continue
185
185
186 parts = []
186 parts = []
187 while line is not None:
187 while line is not None:
188 if line.endswith('\\') and (not has_comment(line)):
188 if line.endswith('\\') and (not has_comment(line)):
189 parts.append(line[:-1])
189 parts.append(line[:-1])
190 line = (yield None) # Get another line
190 line = (yield None) # Get another line
191 else:
191 else:
192 parts.append(line)
192 parts.append(line)
193 break
193 break
194
194
195 # Output
195 # Output
196 line = ''.join(parts)
196 line = ''.join(parts)
197
197
198 # Utilities
198 # Utilities
199 def _make_help_call(target, esc, lspace, next_input=None):
199 def _make_help_call(target, esc, lspace, next_input=None):
200 """Prepares a pinfo(2)/psearch call from a target name and the escape
200 """Prepares a pinfo(2)/psearch call from a target name and the escape
201 (i.e. ? or ??)"""
201 (i.e. ? or ??)"""
202 method = 'pinfo2' if esc == '??' \
202 method = 'pinfo2' if esc == '??' \
203 else 'psearch' if '*' in target \
203 else 'psearch' if '*' in target \
204 else 'pinfo'
204 else 'pinfo'
205 arg = " ".join([method, target])
205 arg = " ".join([method, target])
206 if next_input is None:
206 if next_input is None:
207 return '%sget_ipython().magic(%r)' % (lspace, arg)
207 return '%sget_ipython().magic(%r)' % (lspace, arg)
208 else:
208 else:
209 return '%sget_ipython().set_next_input(%r);get_ipython().magic(%r)' % \
209 return '%sget_ipython().set_next_input(%r);get_ipython().magic(%r)' % \
210 (lspace, next_input, arg)
210 (lspace, next_input, arg)
211
211
212 # These define the transformations for the different escape characters.
212 # These define the transformations for the different escape characters.
213 def _tr_system(line_info):
213 def _tr_system(line_info):
214 "Translate lines escaped with: !"
214 "Translate lines escaped with: !"
215 cmd = line_info.line.lstrip().lstrip(ESC_SHELL)
215 cmd = line_info.line.lstrip().lstrip(ESC_SHELL)
216 return '%sget_ipython().system(%r)' % (line_info.pre, cmd)
216 return '%sget_ipython().system(%r)' % (line_info.pre, cmd)
217
217
218 def _tr_system2(line_info):
218 def _tr_system2(line_info):
219 "Translate lines escaped with: !!"
219 "Translate lines escaped with: !!"
220 cmd = line_info.line.lstrip()[2:]
220 cmd = line_info.line.lstrip()[2:]
221 return '%sget_ipython().getoutput(%r)' % (line_info.pre, cmd)
221 return '%sget_ipython().getoutput(%r)' % (line_info.pre, cmd)
222
222
223 def _tr_help(line_info):
223 def _tr_help(line_info):
224 "Translate lines escaped with: ?/??"
224 "Translate lines escaped with: ?/??"
225 # A naked help line should just fire the intro help screen
225 # A naked help line should just fire the intro help screen
226 if not line_info.line[1:]:
226 if not line_info.line[1:]:
227 return 'get_ipython().show_usage()'
227 return 'get_ipython().show_usage()'
228
228
229 return _make_help_call(line_info.ifun, line_info.esc, line_info.pre)
229 return _make_help_call(line_info.ifun, line_info.esc, line_info.pre)
230
230
231 def _tr_magic(line_info):
231 def _tr_magic(line_info):
232 "Translate lines escaped with: %"
232 "Translate lines escaped with: %"
233 tpl = '%sget_ipython().magic(%r)'
233 tpl = '%sget_ipython().magic(%r)'
234 if line_info.line.startswith(ESC_MAGIC2):
234 if line_info.line.startswith(ESC_MAGIC2):
235 return line_info.line
235 return line_info.line
236 cmd = ' '.join([line_info.ifun, line_info.the_rest]).strip()
236 cmd = ' '.join([line_info.ifun, line_info.the_rest]).strip()
237 return tpl % (line_info.pre, cmd)
237 return tpl % (line_info.pre, cmd)
238
238
239 def _tr_quote(line_info):
239 def _tr_quote(line_info):
240 "Translate lines escaped with: ,"
240 "Translate lines escaped with: ,"
241 return '%s%s("%s")' % (line_info.pre, line_info.ifun,
241 return '%s%s("%s")' % (line_info.pre, line_info.ifun,
242 '", "'.join(line_info.the_rest.split()) )
242 '", "'.join(line_info.the_rest.split()) )
243
243
244 def _tr_quote2(line_info):
244 def _tr_quote2(line_info):
245 "Translate lines escaped with: ;"
245 "Translate lines escaped with: ;"
246 return '%s%s("%s")' % (line_info.pre, line_info.ifun,
246 return '%s%s("%s")' % (line_info.pre, line_info.ifun,
247 line_info.the_rest)
247 line_info.the_rest)
248
248
249 def _tr_paren(line_info):
249 def _tr_paren(line_info):
250 "Translate lines escaped with: /"
250 "Translate lines escaped with: /"
251 return '%s%s(%s)' % (line_info.pre, line_info.ifun,
251 return '%s%s(%s)' % (line_info.pre, line_info.ifun,
252 ", ".join(line_info.the_rest.split()))
252 ", ".join(line_info.the_rest.split()))
253
253
254 tr = { ESC_SHELL : _tr_system,
254 tr = { ESC_SHELL : _tr_system,
255 ESC_SH_CAP : _tr_system2,
255 ESC_SH_CAP : _tr_system2,
256 ESC_HELP : _tr_help,
256 ESC_HELP : _tr_help,
257 ESC_HELP2 : _tr_help,
257 ESC_HELP2 : _tr_help,
258 ESC_MAGIC : _tr_magic,
258 ESC_MAGIC : _tr_magic,
259 ESC_QUOTE : _tr_quote,
259 ESC_QUOTE : _tr_quote,
260 ESC_QUOTE2 : _tr_quote2,
260 ESC_QUOTE2 : _tr_quote2,
261 ESC_PAREN : _tr_paren }
261 ESC_PAREN : _tr_paren }
262
262
263 @StatelessInputTransformer.wrap
263 @StatelessInputTransformer.wrap
264 def escaped_commands(line):
264 def escaped_commands(line):
265 """Transform escaped commands - %magic, !system, ?help + various autocalls.
265 """Transform escaped commands - %magic, !system, ?help + various autocalls.
266 """
266 """
267 if not line or line.isspace():
267 if not line or line.isspace():
268 return line
268 return line
269 lineinf = LineInfo(line)
269 lineinf = LineInfo(line)
270 if lineinf.esc not in tr:
270 if lineinf.esc not in tr:
271 return line
271 return line
272
272
273 return tr[lineinf.esc](lineinf)
273 return tr[lineinf.esc](lineinf)
274
274
275 _initial_space_re = re.compile(r'\s*')
275 _initial_space_re = re.compile(r'\s*')
276
276
277 _help_end_re = re.compile(r"""(%{0,2}
277 _help_end_re = re.compile(r"""(%{0,2}
278 [a-zA-Z_*][\w*]* # Variable name
278 [a-zA-Z_*][\w*]* # Variable name
279 (\.[a-zA-Z_*][\w*]*)* # .etc.etc
279 (\.[a-zA-Z_*][\w*]*)* # .etc.etc
280 )
280 )
281 (\?\??)$ # ? or ??
281 (\?\??)$ # ? or ??
282 """,
282 """,
283 re.VERBOSE)
283 re.VERBOSE)
284
284
285 # Extra pseudotokens for multiline strings and data structures
285 # Extra pseudotokens for multiline strings and data structures
286 _MULTILINE_STRING = object()
286 _MULTILINE_STRING = object()
287 _MULTILINE_STRUCTURE = object()
287 _MULTILINE_STRUCTURE = object()
288
288
289 def _line_tokens(line):
289 def _line_tokens(line):
290 """Helper for has_comment and ends_in_comment_or_string."""
290 """Helper for has_comment and ends_in_comment_or_string."""
291 readline = StringIO(line).readline
291 readline = StringIO(line).readline
292 toktypes = set()
292 toktypes = set()
293 try:
293 try:
294 for t in generate_tokens(readline):
294 for t in generate_tokens(readline):
295 toktypes.add(t[0])
295 toktypes.add(t[0])
296 except TokenError as e:
296 except TokenError as e:
297 # There are only two cases where a TokenError is raised.
297 # There are only two cases where a TokenError is raised.
298 if 'multi-line string' in e.args[0]:
298 if 'multi-line string' in e.args[0]:
299 toktypes.add(_MULTILINE_STRING)
299 toktypes.add(_MULTILINE_STRING)
300 else:
300 else:
301 toktypes.add(_MULTILINE_STRUCTURE)
301 toktypes.add(_MULTILINE_STRUCTURE)
302 return toktypes
302 return toktypes
303
303
304 def has_comment(src):
304 def has_comment(src):
305 """Indicate whether an input line has (i.e. ends in, or is) a comment.
305 """Indicate whether an input line has (i.e. ends in, or is) a comment.
306
306
307 This uses tokenize, so it can distinguish comments from # inside strings.
307 This uses tokenize, so it can distinguish comments from # inside strings.
308
308
309 Parameters
309 Parameters
310 ----------
310 ----------
311 src : string
311 src : string
312 A single line input string.
312 A single line input string.
313
313
314 Returns
314 Returns
315 -------
315 -------
316 comment : bool
316 comment : bool
317 True if source has a comment.
317 True if source has a comment.
318 """
318 """
319 return (tokenize2.COMMENT in _line_tokens(src))
319 return (tokenize2.COMMENT in _line_tokens(src))
320
320
321 def ends_in_comment_or_string(src):
321 def ends_in_comment_or_string(src):
322 """Indicates whether or not an input line ends in a comment or within
322 """Indicates whether or not an input line ends in a comment or within
323 a multiline string.
323 a multiline string.
324
324
325 Parameters
325 Parameters
326 ----------
326 ----------
327 src : string
327 src : string
328 A single line input string.
328 A single line input string.
329
329
330 Returns
330 Returns
331 -------
331 -------
332 comment : bool
332 comment : bool
333 True if source ends in a comment or multiline string.
333 True if source ends in a comment or multiline string.
334 """
334 """
335 toktypes = _line_tokens(src)
335 toktypes = _line_tokens(src)
336 return (tokenize2.COMMENT in toktypes) or (_MULTILINE_STRING in toktypes)
336 return (tokenize2.COMMENT in toktypes) or (_MULTILINE_STRING in toktypes)
337
337
338
338
339 @StatelessInputTransformer.wrap
339 @StatelessInputTransformer.wrap
340 def help_end(line):
340 def help_end(line):
341 """Translate lines with ?/?? at the end"""
341 """Translate lines with ?/?? at the end"""
342 m = _help_end_re.search(line)
342 m = _help_end_re.search(line)
343 if m is None or ends_in_comment_or_string(line):
343 if m is None or ends_in_comment_or_string(line):
344 return line
344 return line
345 target = m.group(1)
345 target = m.group(1)
346 esc = m.group(3)
346 esc = m.group(3)
347 lspace = _initial_space_re.match(line).group(0)
347 lspace = _initial_space_re.match(line).group(0)
348
348
349 # If we're mid-command, put it back on the next prompt for the user.
349 # If we're mid-command, put it back on the next prompt for the user.
350 next_input = line.rstrip('?') if line.strip() != m.group(0) else None
350 next_input = line.rstrip('?') if line.strip() != m.group(0) else None
351
351
352 return _make_help_call(target, esc, lspace, next_input)
352 return _make_help_call(target, esc, lspace, next_input)
353
353
354
354
355 @CoroutineInputTransformer.wrap
355 @CoroutineInputTransformer.wrap
356 def cellmagic(end_on_blank_line=False):
356 def cellmagic(end_on_blank_line=False):
357 """Captures & transforms cell magics.
357 """Captures & transforms cell magics.
358
358
359 After a cell magic is started, this stores up any lines it gets until it is
359 After a cell magic is started, this stores up any lines it gets until it is
360 reset (sent None).
360 reset (sent None).
361 """
361 """
362 tpl = 'get_ipython().run_cell_magic(%r, %r, %r)'
362 tpl = 'get_ipython().run_cell_magic(%r, %r, %r)'
363 cellmagic_help_re = re.compile('%%\w+\?')
363 cellmagic_help_re = re.compile('%%\w+\?')
364 line = ''
364 line = ''
365 while True:
365 while True:
366 line = (yield line)
366 line = (yield line)
367 # consume leading empty lines
367 # consume leading empty lines
368 while not line:
368 while not line:
369 line = (yield line)
369 line = (yield line)
370
370
371 if not line.startswith(ESC_MAGIC2):
371 if not line.startswith(ESC_MAGIC2):
372 # This isn't a cell magic, idle waiting for reset then start over
372 # This isn't a cell magic, idle waiting for reset then start over
373 while line is not None:
373 while line is not None:
374 line = (yield line)
374 line = (yield line)
375 continue
375 continue
376
376
377 if cellmagic_help_re.match(line):
377 if cellmagic_help_re.match(line):
378 # This case will be handled by help_end
378 # This case will be handled by help_end
379 continue
379 continue
380
380
381 first = line
381 first = line
382 body = []
382 body = []
383 line = (yield None)
383 line = (yield None)
384 while (line is not None) and \
384 while (line is not None) and \
385 ((line.strip() != '') or not end_on_blank_line):
385 ((line.strip() != '') or not end_on_blank_line):
386 body.append(line)
386 body.append(line)
387 line = (yield None)
387 line = (yield None)
388
388
389 # Output
389 # Output
390 magic_name, _, first = first.partition(' ')
390 magic_name, _, first = first.partition(' ')
391 magic_name = magic_name.lstrip(ESC_MAGIC2)
391 magic_name = magic_name.lstrip(ESC_MAGIC2)
392 line = tpl % (magic_name, first, u'\n'.join(body))
392 line = tpl % (magic_name, first, u'\n'.join(body))
393
393
394
394
395 def _strip_prompts(prompt_re, initial_re=None):
395 def _strip_prompts(prompt_re, initial_re=None):
396 """Remove matching input prompts from a block of input.
396 """Remove matching input prompts from a block of input.
397
397
398 Parameters
398 Parameters
399 ----------
399 ----------
400 prompt_re : regular expression
400 prompt_re : regular expression
401 A regular expression matching any input prompt (including continuation)
401 A regular expression matching any input prompt (including continuation)
402 initial_re : regular expression, optional
402 initial_re : regular expression, optional
403 A regular expression matching only the initial prompt, but not continuation.
403 A regular expression matching only the initial prompt, but not continuation.
404 If no initial expression is given, prompt_re will be used everywhere.
404 If no initial expression is given, prompt_re will be used everywhere.
405 Used mainly for plain Python prompts, where the continuation prompt
405 Used mainly for plain Python prompts, where the continuation prompt
406 ``...`` is a valid Python expression in Python 3, so shouldn't be stripped.
406 ``...`` is a valid Python expression in Python 3, so shouldn't be stripped.
407
407
408 If initial_re and prompt_re differ,
408 If initial_re and prompt_re differ,
409 only initial_re will be tested against the first line.
409 only initial_re will be tested against the first line.
410 If any prompt is found on the first two lines,
410 If any prompt is found on the first two lines,
411 prompts will be stripped from the rest of the block.
411 prompts will be stripped from the rest of the block.
412 """
412 """
413 if initial_re is None:
413 if initial_re is None:
414 initial_re = prompt_re
414 initial_re = prompt_re
415 line = ''
415 line = ''
416 while True:
416 while True:
417 line = (yield line)
417 line = (yield line)
418
418
419 # First line of cell
419 # First line of cell
420 if line is None:
420 if line is None:
421 continue
421 continue
422 out, n1 = initial_re.subn('', line, count=1)
422 out, n1 = initial_re.subn('', line, count=1)
423 line = (yield out)
423 line = (yield out)
424
424
425 if line is None:
425 if line is None:
426 continue
426 continue
427 # check for any prompt on the second line of the cell,
427 # check for any prompt on the second line of the cell,
428 # because people often copy from just after the first prompt,
428 # because people often copy from just after the first prompt,
429 # so we might not see it in the first line.
429 # so we might not see it in the first line.
430 out, n2 = prompt_re.subn('', line, count=1)
430 out, n2 = prompt_re.subn('', line, count=1)
431 line = (yield out)
431 line = (yield out)
432
432
433 if n1 or n2:
433 if n1 or n2:
434 # Found a prompt in the first two lines - check for it in
434 # Found a prompt in the first two lines - check for it in
435 # the rest of the cell as well.
435 # the rest of the cell as well.
436 while line is not None:
436 while line is not None:
437 line = (yield prompt_re.sub('', line, count=1))
437 line = (yield prompt_re.sub('', line, count=1))
438
438
439 else:
439 else:
440 # Prompts not in input - wait for reset
440 # Prompts not in input - wait for reset
441 while line is not None:
441 while line is not None:
442 line = (yield line)
442 line = (yield line)
443
443
444 @CoroutineInputTransformer.wrap
444 @CoroutineInputTransformer.wrap
445 def classic_prompt():
445 def classic_prompt():
446 """Strip the >>>/... prompts of the Python interactive shell."""
446 """Strip the >>>/... prompts of the Python interactive shell."""
447 # FIXME: non-capturing version (?:...) usable?
447 # FIXME: non-capturing version (?:...) usable?
448 prompt_re = re.compile(r'^(>>> ?|\.\.\. ?)')
448 prompt_re = re.compile(r'^(>>> ?|\.\.\. ?)')
449 initial_re = re.compile(r'^(>>> ?)')
449 initial_re = re.compile(r'^(>>> ?)')
450 return _strip_prompts(prompt_re, initial_re)
450 return _strip_prompts(prompt_re, initial_re)
451
451
452 @CoroutineInputTransformer.wrap
452 @CoroutineInputTransformer.wrap
453 def ipy_prompt():
453 def ipy_prompt():
454 """Strip IPython's In [1]:/...: prompts."""
454 """Strip IPython's In [1]:/...: prompts."""
455 # FIXME: non-capturing version (?:...) usable?
455 # FIXME: non-capturing version (?:...) usable?
456 # FIXME: r'^(In \[\d+\]: | {3}\.{3,}: )' clearer?
456 prompt_re = re.compile(r'^(In \[\d+\]: |\ {3,}\.{3,}: )')
457 prompt_re = re.compile(r'^(In \[\d+\]: |\ \ \ \.\.\.+: )')
458 return _strip_prompts(prompt_re)
457 return _strip_prompts(prompt_re)
459
458
460
459
461 @CoroutineInputTransformer.wrap
460 @CoroutineInputTransformer.wrap
462 def leading_indent():
461 def leading_indent():
463 """Remove leading indentation.
462 """Remove leading indentation.
464
463
465 If the first line starts with a spaces or tabs, the same whitespace will be
464 If the first line starts with a spaces or tabs, the same whitespace will be
466 removed from each following line until it is reset.
465 removed from each following line until it is reset.
467 """
466 """
468 space_re = re.compile(r'^[ \t]+')
467 space_re = re.compile(r'^[ \t]+')
469 line = ''
468 line = ''
470 while True:
469 while True:
471 line = (yield line)
470 line = (yield line)
472
471
473 if line is None:
472 if line is None:
474 continue
473 continue
475
474
476 m = space_re.match(line)
475 m = space_re.match(line)
477 if m:
476 if m:
478 space = m.group(0)
477 space = m.group(0)
479 while line is not None:
478 while line is not None:
480 if line.startswith(space):
479 if line.startswith(space):
481 line = line[len(space):]
480 line = line[len(space):]
482 line = (yield line)
481 line = (yield line)
483 else:
482 else:
484 # No leading spaces - wait for reset
483 # No leading spaces - wait for reset
485 while line is not None:
484 while line is not None:
486 line = (yield line)
485 line = (yield line)
487
486
488
487
489 @CoroutineInputTransformer.wrap
488 @CoroutineInputTransformer.wrap
490 def strip_encoding_cookie():
489 def strip_encoding_cookie():
491 """Remove encoding comment if found in first two lines
490 """Remove encoding comment if found in first two lines
492
491
493 If the first or second line has the `# coding: utf-8` comment,
492 If the first or second line has the `# coding: utf-8` comment,
494 it will be removed.
493 it will be removed.
495 """
494 """
496 line = ''
495 line = ''
497 while True:
496 while True:
498 line = (yield line)
497 line = (yield line)
499 # check comment on first two lines
498 # check comment on first two lines
500 for i in range(2):
499 for i in range(2):
501 if line is None:
500 if line is None:
502 break
501 break
503 if cookie_comment_re.match(line):
502 if cookie_comment_re.match(line):
504 line = (yield "")
503 line = (yield "")
505 else:
504 else:
506 line = (yield line)
505 line = (yield line)
507
506
508 # no-op on the rest of the cell
507 # no-op on the rest of the cell
509 while line is not None:
508 while line is not None:
510 line = (yield line)
509 line = (yield line)
511
510
512
511
513 assign_system_re = re.compile(r'(?P<lhs>(\s*)([\w\.]+)((\s*,\s*[\w\.]+)*))'
512 assign_system_re = re.compile(r'(?P<lhs>(\s*)([\w\.]+)((\s*,\s*[\w\.]+)*))'
514 r'\s*=\s*!\s*(?P<cmd>.*)')
513 r'\s*=\s*!\s*(?P<cmd>.*)')
515 assign_system_template = '%s = get_ipython().getoutput(%r)'
514 assign_system_template = '%s = get_ipython().getoutput(%r)'
516 @StatelessInputTransformer.wrap
515 @StatelessInputTransformer.wrap
517 def assign_from_system(line):
516 def assign_from_system(line):
518 """Transform assignment from system commands (e.g. files = !ls)"""
517 """Transform assignment from system commands (e.g. files = !ls)"""
519 m = assign_system_re.match(line)
518 m = assign_system_re.match(line)
520 if m is None:
519 if m is None:
521 return line
520 return line
522
521
523 return assign_system_template % m.group('lhs', 'cmd')
522 return assign_system_template % m.group('lhs', 'cmd')
524
523
525 assign_magic_re = re.compile(r'(?P<lhs>(\s*)([\w\.]+)((\s*,\s*[\w\.]+)*))'
524 assign_magic_re = re.compile(r'(?P<lhs>(\s*)([\w\.]+)((\s*,\s*[\w\.]+)*))'
526 r'\s*=\s*%\s*(?P<cmd>.*)')
525 r'\s*=\s*%\s*(?P<cmd>.*)')
527 assign_magic_template = '%s = get_ipython().magic(%r)'
526 assign_magic_template = '%s = get_ipython().magic(%r)'
528 @StatelessInputTransformer.wrap
527 @StatelessInputTransformer.wrap
529 def assign_from_magic(line):
528 def assign_from_magic(line):
530 """Transform assignment from magic commands (e.g. a = %who_ls)"""
529 """Transform assignment from magic commands (e.g. a = %who_ls)"""
531 m = assign_magic_re.match(line)
530 m = assign_magic_re.match(line)
532 if m is None:
531 if m is None:
533 return line
532 return line
534
533
535 return assign_magic_template % m.group('lhs', 'cmd')
534 return assign_magic_template % m.group('lhs', 'cmd')
General Comments 0
You need to be logged in to leave comments. Login now