##// END OF EJS Templates
Improve assignment regex to match more tuple unpacking syntax...
Thomas Kluyver -
Show More
@@ -1,542 +1,549 b''
1 """Input transformer classes to support IPython special syntax.
1 """Input transformer classes to support IPython special syntax.
2
2
3 This includes the machinery to recognise and transform ``%magic`` commands,
3 This includes the machinery to recognise and transform ``%magic`` commands,
4 ``!system`` commands, ``help?`` querying, prompt stripping, and so forth.
4 ``!system`` commands, ``help?`` querying, prompt stripping, and so forth.
5 """
5 """
6 import abc
6 import abc
7 import functools
7 import functools
8 import re
8 import re
9
9
10 from IPython.core.splitinput import LineInfo
10 from IPython.core.splitinput import LineInfo
11 from IPython.utils import tokenize2
11 from IPython.utils import tokenize2
12 from IPython.utils.openpy import cookie_comment_re
12 from IPython.utils.openpy import cookie_comment_re
13 from IPython.utils.py3compat import with_metaclass, PY3
13 from IPython.utils.py3compat import with_metaclass, PY3
14 from IPython.utils.tokenize2 import generate_tokens, untokenize, TokenError
14 from IPython.utils.tokenize2 import generate_tokens, untokenize, TokenError
15
15
16 if PY3:
16 if PY3:
17 from io import StringIO
17 from io import StringIO
18 else:
18 else:
19 from StringIO import StringIO
19 from StringIO import StringIO
20
20
21 #-----------------------------------------------------------------------------
21 #-----------------------------------------------------------------------------
22 # Globals
22 # Globals
23 #-----------------------------------------------------------------------------
23 #-----------------------------------------------------------------------------
24
24
25 # The escape sequences that define the syntax transformations IPython will
25 # The escape sequences that define the syntax transformations IPython will
26 # apply to user input. These can NOT be just changed here: many regular
26 # apply to user input. These can NOT be just changed here: many regular
27 # expressions and other parts of the code may use their hardcoded values, and
27 # expressions and other parts of the code may use their hardcoded values, and
28 # for all intents and purposes they constitute the 'IPython syntax', so they
28 # for all intents and purposes they constitute the 'IPython syntax', so they
29 # should be considered fixed.
29 # should be considered fixed.
30
30
31 ESC_SHELL = '!' # Send line to underlying system shell
31 ESC_SHELL = '!' # Send line to underlying system shell
32 ESC_SH_CAP = '!!' # Send line to system shell and capture output
32 ESC_SH_CAP = '!!' # Send line to system shell and capture output
33 ESC_HELP = '?' # Find information about object
33 ESC_HELP = '?' # Find information about object
34 ESC_HELP2 = '??' # Find extra-detailed information about object
34 ESC_HELP2 = '??' # Find extra-detailed information about object
35 ESC_MAGIC = '%' # Call magic function
35 ESC_MAGIC = '%' # Call magic function
36 ESC_MAGIC2 = '%%' # Call cell-magic function
36 ESC_MAGIC2 = '%%' # Call cell-magic function
37 ESC_QUOTE = ',' # Split args on whitespace, quote each as string and call
37 ESC_QUOTE = ',' # Split args on whitespace, quote each as string and call
38 ESC_QUOTE2 = ';' # Quote all args as a single string, call
38 ESC_QUOTE2 = ';' # Quote all args as a single string, call
39 ESC_PAREN = '/' # Call first argument with rest of line as arguments
39 ESC_PAREN = '/' # Call first argument with rest of line as arguments
40
40
41 ESC_SEQUENCES = [ESC_SHELL, ESC_SH_CAP, ESC_HELP ,\
41 ESC_SEQUENCES = [ESC_SHELL, ESC_SH_CAP, ESC_HELP ,\
42 ESC_HELP2, ESC_MAGIC, ESC_MAGIC2,\
42 ESC_HELP2, ESC_MAGIC, ESC_MAGIC2,\
43 ESC_QUOTE, ESC_QUOTE2, ESC_PAREN ]
43 ESC_QUOTE, ESC_QUOTE2, ESC_PAREN ]
44
44
45
45
46 class InputTransformer(with_metaclass(abc.ABCMeta, object)):
46 class InputTransformer(with_metaclass(abc.ABCMeta, object)):
47 """Abstract base class for line-based input transformers."""
47 """Abstract base class for line-based input transformers."""
48
48
49 @abc.abstractmethod
49 @abc.abstractmethod
50 def push(self, line):
50 def push(self, line):
51 """Send a line of input to the transformer, returning the transformed
51 """Send a line of input to the transformer, returning the transformed
52 input or None if the transformer is waiting for more input.
52 input or None if the transformer is waiting for more input.
53
53
54 Must be overridden by subclasses.
54 Must be overridden by subclasses.
55
55
56 Implementations may raise ``SyntaxError`` if the input is invalid. No
56 Implementations may raise ``SyntaxError`` if the input is invalid. No
57 other exceptions may be raised.
57 other exceptions may be raised.
58 """
58 """
59 pass
59 pass
60
60
61 @abc.abstractmethod
61 @abc.abstractmethod
62 def reset(self):
62 def reset(self):
63 """Return, transformed any lines that the transformer has accumulated,
63 """Return, transformed any lines that the transformer has accumulated,
64 and reset its internal state.
64 and reset its internal state.
65
65
66 Must be overridden by subclasses.
66 Must be overridden by subclasses.
67 """
67 """
68 pass
68 pass
69
69
70 @classmethod
70 @classmethod
71 def wrap(cls, func):
71 def wrap(cls, func):
72 """Can be used by subclasses as a decorator, to return a factory that
72 """Can be used by subclasses as a decorator, to return a factory that
73 will allow instantiation with the decorated object.
73 will allow instantiation with the decorated object.
74 """
74 """
75 @functools.wraps(func)
75 @functools.wraps(func)
76 def transformer_factory(**kwargs):
76 def transformer_factory(**kwargs):
77 return cls(func, **kwargs)
77 return cls(func, **kwargs)
78
78
79 return transformer_factory
79 return transformer_factory
80
80
81 class StatelessInputTransformer(InputTransformer):
81 class StatelessInputTransformer(InputTransformer):
82 """Wrapper for a stateless input transformer implemented as a function."""
82 """Wrapper for a stateless input transformer implemented as a function."""
83 def __init__(self, func):
83 def __init__(self, func):
84 self.func = func
84 self.func = func
85
85
86 def __repr__(self):
86 def __repr__(self):
87 return "StatelessInputTransformer(func={0!r})".format(self.func)
87 return "StatelessInputTransformer(func={0!r})".format(self.func)
88
88
89 def push(self, line):
89 def push(self, line):
90 """Send a line of input to the transformer, returning the
90 """Send a line of input to the transformer, returning the
91 transformed input."""
91 transformed input."""
92 return self.func(line)
92 return self.func(line)
93
93
94 def reset(self):
94 def reset(self):
95 """No-op - exists for compatibility."""
95 """No-op - exists for compatibility."""
96 pass
96 pass
97
97
98 class CoroutineInputTransformer(InputTransformer):
98 class CoroutineInputTransformer(InputTransformer):
99 """Wrapper for an input transformer implemented as a coroutine."""
99 """Wrapper for an input transformer implemented as a coroutine."""
100 def __init__(self, coro, **kwargs):
100 def __init__(self, coro, **kwargs):
101 # Prime it
101 # Prime it
102 self.coro = coro(**kwargs)
102 self.coro = coro(**kwargs)
103 next(self.coro)
103 next(self.coro)
104
104
105 def __repr__(self):
105 def __repr__(self):
106 return "CoroutineInputTransformer(coro={0!r})".format(self.coro)
106 return "CoroutineInputTransformer(coro={0!r})".format(self.coro)
107
107
108 def push(self, line):
108 def push(self, line):
109 """Send a line of input to the transformer, returning the
109 """Send a line of input to the transformer, returning the
110 transformed input or None if the transformer is waiting for more
110 transformed input or None if the transformer is waiting for more
111 input.
111 input.
112 """
112 """
113 return self.coro.send(line)
113 return self.coro.send(line)
114
114
115 def reset(self):
115 def reset(self):
116 """Return, transformed any lines that the transformer has
116 """Return, transformed any lines that the transformer has
117 accumulated, and reset its internal state.
117 accumulated, and reset its internal state.
118 """
118 """
119 return self.coro.send(None)
119 return self.coro.send(None)
120
120
121 class TokenInputTransformer(InputTransformer):
121 class TokenInputTransformer(InputTransformer):
122 """Wrapper for a token-based input transformer.
122 """Wrapper for a token-based input transformer.
123
123
124 func should accept a list of tokens (5-tuples, see tokenize docs), and
124 func should accept a list of tokens (5-tuples, see tokenize docs), and
125 return an iterable which can be passed to tokenize.untokenize().
125 return an iterable which can be passed to tokenize.untokenize().
126 """
126 """
127 def __init__(self, func):
127 def __init__(self, func):
128 self.func = func
128 self.func = func
129 self.current_line = ""
129 self.current_line = ""
130 self.line_used = False
130 self.line_used = False
131 self.reset_tokenizer()
131 self.reset_tokenizer()
132
132
133 def reset_tokenizer(self):
133 def reset_tokenizer(self):
134 self.tokenizer = generate_tokens(self.get_line)
134 self.tokenizer = generate_tokens(self.get_line)
135
135
136 def get_line(self):
136 def get_line(self):
137 if self.line_used:
137 if self.line_used:
138 raise TokenError
138 raise TokenError
139 self.line_used = True
139 self.line_used = True
140 return self.current_line
140 return self.current_line
141
141
142 def push(self, line):
142 def push(self, line):
143 self.current_line += line + "\n"
143 self.current_line += line + "\n"
144 if self.current_line.isspace():
144 if self.current_line.isspace():
145 return self.reset()
145 return self.reset()
146
146
147 self.line_used = False
147 self.line_used = False
148 tokens = []
148 tokens = []
149 stop_at_NL = False
149 stop_at_NL = False
150 try:
150 try:
151 for intok in self.tokenizer:
151 for intok in self.tokenizer:
152 tokens.append(intok)
152 tokens.append(intok)
153 t = intok[0]
153 t = intok[0]
154 if t == tokenize2.NEWLINE or (stop_at_NL and t == tokenize2.NL):
154 if t == tokenize2.NEWLINE or (stop_at_NL and t == tokenize2.NL):
155 # Stop before we try to pull a line we don't have yet
155 # Stop before we try to pull a line we don't have yet
156 break
156 break
157 elif t == tokenize2.ERRORTOKEN:
157 elif t == tokenize2.ERRORTOKEN:
158 stop_at_NL = True
158 stop_at_NL = True
159 except TokenError:
159 except TokenError:
160 # Multi-line statement - stop and try again with the next line
160 # Multi-line statement - stop and try again with the next line
161 self.reset_tokenizer()
161 self.reset_tokenizer()
162 return None
162 return None
163
163
164 return self.output(tokens)
164 return self.output(tokens)
165
165
166 def output(self, tokens):
166 def output(self, tokens):
167 self.current_line = ""
167 self.current_line = ""
168 self.reset_tokenizer()
168 self.reset_tokenizer()
169 return untokenize(self.func(tokens)).rstrip('\n')
169 return untokenize(self.func(tokens)).rstrip('\n')
170
170
171 def reset(self):
171 def reset(self):
172 l = self.current_line
172 l = self.current_line
173 self.current_line = ""
173 self.current_line = ""
174 self.reset_tokenizer()
174 self.reset_tokenizer()
175 if l:
175 if l:
176 return l.rstrip('\n')
176 return l.rstrip('\n')
177
177
178 class assemble_python_lines(TokenInputTransformer):
178 class assemble_python_lines(TokenInputTransformer):
179 def __init__(self):
179 def __init__(self):
180 super(assemble_python_lines, self).__init__(None)
180 super(assemble_python_lines, self).__init__(None)
181
181
182 def output(self, tokens):
182 def output(self, tokens):
183 return self.reset()
183 return self.reset()
184
184
185 @CoroutineInputTransformer.wrap
185 @CoroutineInputTransformer.wrap
186 def assemble_logical_lines():
186 def assemble_logical_lines():
187 """Join lines following explicit line continuations (\)"""
187 """Join lines following explicit line continuations (\)"""
188 line = ''
188 line = ''
189 while True:
189 while True:
190 line = (yield line)
190 line = (yield line)
191 if not line or line.isspace():
191 if not line or line.isspace():
192 continue
192 continue
193
193
194 parts = []
194 parts = []
195 while line is not None:
195 while line is not None:
196 if line.endswith('\\') and (not has_comment(line)):
196 if line.endswith('\\') and (not has_comment(line)):
197 parts.append(line[:-1])
197 parts.append(line[:-1])
198 line = (yield None) # Get another line
198 line = (yield None) # Get another line
199 else:
199 else:
200 parts.append(line)
200 parts.append(line)
201 break
201 break
202
202
203 # Output
203 # Output
204 line = ''.join(parts)
204 line = ''.join(parts)
205
205
206 # Utilities
206 # Utilities
207 def _make_help_call(target, esc, lspace, next_input=None):
207 def _make_help_call(target, esc, lspace, next_input=None):
208 """Prepares a pinfo(2)/psearch call from a target name and the escape
208 """Prepares a pinfo(2)/psearch call from a target name and the escape
209 (i.e. ? or ??)"""
209 (i.e. ? or ??)"""
210 method = 'pinfo2' if esc == '??' \
210 method = 'pinfo2' if esc == '??' \
211 else 'psearch' if '*' in target \
211 else 'psearch' if '*' in target \
212 else 'pinfo'
212 else 'pinfo'
213 arg = " ".join([method, target])
213 arg = " ".join([method, target])
214 if next_input is None:
214 if next_input is None:
215 return '%sget_ipython().magic(%r)' % (lspace, arg)
215 return '%sget_ipython().magic(%r)' % (lspace, arg)
216 else:
216 else:
217 return '%sget_ipython().set_next_input(%r);get_ipython().magic(%r)' % \
217 return '%sget_ipython().set_next_input(%r);get_ipython().magic(%r)' % \
218 (lspace, next_input, arg)
218 (lspace, next_input, arg)
219
219
220 # These define the transformations for the different escape characters.
220 # These define the transformations for the different escape characters.
221 def _tr_system(line_info):
221 def _tr_system(line_info):
222 "Translate lines escaped with: !"
222 "Translate lines escaped with: !"
223 cmd = line_info.line.lstrip().lstrip(ESC_SHELL)
223 cmd = line_info.line.lstrip().lstrip(ESC_SHELL)
224 return '%sget_ipython().system(%r)' % (line_info.pre, cmd)
224 return '%sget_ipython().system(%r)' % (line_info.pre, cmd)
225
225
226 def _tr_system2(line_info):
226 def _tr_system2(line_info):
227 "Translate lines escaped with: !!"
227 "Translate lines escaped with: !!"
228 cmd = line_info.line.lstrip()[2:]
228 cmd = line_info.line.lstrip()[2:]
229 return '%sget_ipython().getoutput(%r)' % (line_info.pre, cmd)
229 return '%sget_ipython().getoutput(%r)' % (line_info.pre, cmd)
230
230
231 def _tr_help(line_info):
231 def _tr_help(line_info):
232 "Translate lines escaped with: ?/??"
232 "Translate lines escaped with: ?/??"
233 # A naked help line should just fire the intro help screen
233 # A naked help line should just fire the intro help screen
234 if not line_info.line[1:]:
234 if not line_info.line[1:]:
235 return 'get_ipython().show_usage()'
235 return 'get_ipython().show_usage()'
236
236
237 return _make_help_call(line_info.ifun, line_info.esc, line_info.pre)
237 return _make_help_call(line_info.ifun, line_info.esc, line_info.pre)
238
238
239 def _tr_magic(line_info):
239 def _tr_magic(line_info):
240 "Translate lines escaped with: %"
240 "Translate lines escaped with: %"
241 tpl = '%sget_ipython().magic(%r)'
241 tpl = '%sget_ipython().magic(%r)'
242 if line_info.line.startswith(ESC_MAGIC2):
242 if line_info.line.startswith(ESC_MAGIC2):
243 return line_info.line
243 return line_info.line
244 cmd = ' '.join([line_info.ifun, line_info.the_rest]).strip()
244 cmd = ' '.join([line_info.ifun, line_info.the_rest]).strip()
245 return tpl % (line_info.pre, cmd)
245 return tpl % (line_info.pre, cmd)
246
246
247 def _tr_quote(line_info):
247 def _tr_quote(line_info):
248 "Translate lines escaped with: ,"
248 "Translate lines escaped with: ,"
249 return '%s%s("%s")' % (line_info.pre, line_info.ifun,
249 return '%s%s("%s")' % (line_info.pre, line_info.ifun,
250 '", "'.join(line_info.the_rest.split()) )
250 '", "'.join(line_info.the_rest.split()) )
251
251
252 def _tr_quote2(line_info):
252 def _tr_quote2(line_info):
253 "Translate lines escaped with: ;"
253 "Translate lines escaped with: ;"
254 return '%s%s("%s")' % (line_info.pre, line_info.ifun,
254 return '%s%s("%s")' % (line_info.pre, line_info.ifun,
255 line_info.the_rest)
255 line_info.the_rest)
256
256
257 def _tr_paren(line_info):
257 def _tr_paren(line_info):
258 "Translate lines escaped with: /"
258 "Translate lines escaped with: /"
259 return '%s%s(%s)' % (line_info.pre, line_info.ifun,
259 return '%s%s(%s)' % (line_info.pre, line_info.ifun,
260 ", ".join(line_info.the_rest.split()))
260 ", ".join(line_info.the_rest.split()))
261
261
262 tr = { ESC_SHELL : _tr_system,
262 tr = { ESC_SHELL : _tr_system,
263 ESC_SH_CAP : _tr_system2,
263 ESC_SH_CAP : _tr_system2,
264 ESC_HELP : _tr_help,
264 ESC_HELP : _tr_help,
265 ESC_HELP2 : _tr_help,
265 ESC_HELP2 : _tr_help,
266 ESC_MAGIC : _tr_magic,
266 ESC_MAGIC : _tr_magic,
267 ESC_QUOTE : _tr_quote,
267 ESC_QUOTE : _tr_quote,
268 ESC_QUOTE2 : _tr_quote2,
268 ESC_QUOTE2 : _tr_quote2,
269 ESC_PAREN : _tr_paren }
269 ESC_PAREN : _tr_paren }
270
270
271 @StatelessInputTransformer.wrap
271 @StatelessInputTransformer.wrap
272 def escaped_commands(line):
272 def escaped_commands(line):
273 """Transform escaped commands - %magic, !system, ?help + various autocalls.
273 """Transform escaped commands - %magic, !system, ?help + various autocalls.
274 """
274 """
275 if not line or line.isspace():
275 if not line or line.isspace():
276 return line
276 return line
277 lineinf = LineInfo(line)
277 lineinf = LineInfo(line)
278 if lineinf.esc not in tr:
278 if lineinf.esc not in tr:
279 return line
279 return line
280
280
281 return tr[lineinf.esc](lineinf)
281 return tr[lineinf.esc](lineinf)
282
282
283 _initial_space_re = re.compile(r'\s*')
283 _initial_space_re = re.compile(r'\s*')
284
284
285 _help_end_re = re.compile(r"""(%{0,2}
285 _help_end_re = re.compile(r"""(%{0,2}
286 [a-zA-Z_*][\w*]* # Variable name
286 [a-zA-Z_*][\w*]* # Variable name
287 (\.[a-zA-Z_*][\w*]*)* # .etc.etc
287 (\.[a-zA-Z_*][\w*]*)* # .etc.etc
288 )
288 )
289 (\?\??)$ # ? or ??
289 (\?\??)$ # ? or ??
290 """,
290 """,
291 re.VERBOSE)
291 re.VERBOSE)
292
292
293 # Extra pseudotokens for multiline strings and data structures
293 # Extra pseudotokens for multiline strings and data structures
294 _MULTILINE_STRING = object()
294 _MULTILINE_STRING = object()
295 _MULTILINE_STRUCTURE = object()
295 _MULTILINE_STRUCTURE = object()
296
296
297 def _line_tokens(line):
297 def _line_tokens(line):
298 """Helper for has_comment and ends_in_comment_or_string."""
298 """Helper for has_comment and ends_in_comment_or_string."""
299 readline = StringIO(line).readline
299 readline = StringIO(line).readline
300 toktypes = set()
300 toktypes = set()
301 try:
301 try:
302 for t in generate_tokens(readline):
302 for t in generate_tokens(readline):
303 toktypes.add(t[0])
303 toktypes.add(t[0])
304 except TokenError as e:
304 except TokenError as e:
305 # There are only two cases where a TokenError is raised.
305 # There are only two cases where a TokenError is raised.
306 if 'multi-line string' in e.args[0]:
306 if 'multi-line string' in e.args[0]:
307 toktypes.add(_MULTILINE_STRING)
307 toktypes.add(_MULTILINE_STRING)
308 else:
308 else:
309 toktypes.add(_MULTILINE_STRUCTURE)
309 toktypes.add(_MULTILINE_STRUCTURE)
310 return toktypes
310 return toktypes
311
311
312 def has_comment(src):
312 def has_comment(src):
313 """Indicate whether an input line has (i.e. ends in, or is) a comment.
313 """Indicate whether an input line has (i.e. ends in, or is) a comment.
314
314
315 This uses tokenize, so it can distinguish comments from # inside strings.
315 This uses tokenize, so it can distinguish comments from # inside strings.
316
316
317 Parameters
317 Parameters
318 ----------
318 ----------
319 src : string
319 src : string
320 A single line input string.
320 A single line input string.
321
321
322 Returns
322 Returns
323 -------
323 -------
324 comment : bool
324 comment : bool
325 True if source has a comment.
325 True if source has a comment.
326 """
326 """
327 return (tokenize2.COMMENT in _line_tokens(src))
327 return (tokenize2.COMMENT in _line_tokens(src))
328
328
329 def ends_in_comment_or_string(src):
329 def ends_in_comment_or_string(src):
330 """Indicates whether or not an input line ends in a comment or within
330 """Indicates whether or not an input line ends in a comment or within
331 a multiline string.
331 a multiline string.
332
332
333 Parameters
333 Parameters
334 ----------
334 ----------
335 src : string
335 src : string
336 A single line input string.
336 A single line input string.
337
337
338 Returns
338 Returns
339 -------
339 -------
340 comment : bool
340 comment : bool
341 True if source ends in a comment or multiline string.
341 True if source ends in a comment or multiline string.
342 """
342 """
343 toktypes = _line_tokens(src)
343 toktypes = _line_tokens(src)
344 return (tokenize2.COMMENT in toktypes) or (_MULTILINE_STRING in toktypes)
344 return (tokenize2.COMMENT in toktypes) or (_MULTILINE_STRING in toktypes)
345
345
346
346
347 @StatelessInputTransformer.wrap
347 @StatelessInputTransformer.wrap
348 def help_end(line):
348 def help_end(line):
349 """Translate lines with ?/?? at the end"""
349 """Translate lines with ?/?? at the end"""
350 m = _help_end_re.search(line)
350 m = _help_end_re.search(line)
351 if m is None or ends_in_comment_or_string(line):
351 if m is None or ends_in_comment_or_string(line):
352 return line
352 return line
353 target = m.group(1)
353 target = m.group(1)
354 esc = m.group(3)
354 esc = m.group(3)
355 lspace = _initial_space_re.match(line).group(0)
355 lspace = _initial_space_re.match(line).group(0)
356
356
357 # If we're mid-command, put it back on the next prompt for the user.
357 # If we're mid-command, put it back on the next prompt for the user.
358 next_input = line.rstrip('?') if line.strip() != m.group(0) else None
358 next_input = line.rstrip('?') if line.strip() != m.group(0) else None
359
359
360 return _make_help_call(target, esc, lspace, next_input)
360 return _make_help_call(target, esc, lspace, next_input)
361
361
362
362
363 @CoroutineInputTransformer.wrap
363 @CoroutineInputTransformer.wrap
364 def cellmagic(end_on_blank_line=False):
364 def cellmagic(end_on_blank_line=False):
365 """Captures & transforms cell magics.
365 """Captures & transforms cell magics.
366
366
367 After a cell magic is started, this stores up any lines it gets until it is
367 After a cell magic is started, this stores up any lines it gets until it is
368 reset (sent None).
368 reset (sent None).
369 """
369 """
370 tpl = 'get_ipython().run_cell_magic(%r, %r, %r)'
370 tpl = 'get_ipython().run_cell_magic(%r, %r, %r)'
371 cellmagic_help_re = re.compile('%%\w+\?')
371 cellmagic_help_re = re.compile('%%\w+\?')
372 line = ''
372 line = ''
373 while True:
373 while True:
374 line = (yield line)
374 line = (yield line)
375 # consume leading empty lines
375 # consume leading empty lines
376 while not line:
376 while not line:
377 line = (yield line)
377 line = (yield line)
378
378
379 if not line.startswith(ESC_MAGIC2):
379 if not line.startswith(ESC_MAGIC2):
380 # This isn't a cell magic, idle waiting for reset then start over
380 # This isn't a cell magic, idle waiting for reset then start over
381 while line is not None:
381 while line is not None:
382 line = (yield line)
382 line = (yield line)
383 continue
383 continue
384
384
385 if cellmagic_help_re.match(line):
385 if cellmagic_help_re.match(line):
386 # This case will be handled by help_end
386 # This case will be handled by help_end
387 continue
387 continue
388
388
389 first = line
389 first = line
390 body = []
390 body = []
391 line = (yield None)
391 line = (yield None)
392 while (line is not None) and \
392 while (line is not None) and \
393 ((line.strip() != '') or not end_on_blank_line):
393 ((line.strip() != '') or not end_on_blank_line):
394 body.append(line)
394 body.append(line)
395 line = (yield None)
395 line = (yield None)
396
396
397 # Output
397 # Output
398 magic_name, _, first = first.partition(' ')
398 magic_name, _, first = first.partition(' ')
399 magic_name = magic_name.lstrip(ESC_MAGIC2)
399 magic_name = magic_name.lstrip(ESC_MAGIC2)
400 line = tpl % (magic_name, first, u'\n'.join(body))
400 line = tpl % (magic_name, first, u'\n'.join(body))
401
401
402
402
403 def _strip_prompts(prompt_re, initial_re=None):
403 def _strip_prompts(prompt_re, initial_re=None):
404 """Remove matching input prompts from a block of input.
404 """Remove matching input prompts from a block of input.
405
405
406 Parameters
406 Parameters
407 ----------
407 ----------
408 prompt_re : regular expression
408 prompt_re : regular expression
409 A regular expression matching any input prompt (including continuation)
409 A regular expression matching any input prompt (including continuation)
410 initial_re : regular expression, optional
410 initial_re : regular expression, optional
411 A regular expression matching only the initial prompt, but not continuation.
411 A regular expression matching only the initial prompt, but not continuation.
412 If no initial expression is given, prompt_re will be used everywhere.
412 If no initial expression is given, prompt_re will be used everywhere.
413 Used mainly for plain Python prompts, where the continuation prompt
413 Used mainly for plain Python prompts, where the continuation prompt
414 ``...`` is a valid Python expression in Python 3, so shouldn't be stripped.
414 ``...`` is a valid Python expression in Python 3, so shouldn't be stripped.
415
415
416 If initial_re and prompt_re differ,
416 If initial_re and prompt_re differ,
417 only initial_re will be tested against the first line.
417 only initial_re will be tested against the first line.
418 If any prompt is found on the first two lines,
418 If any prompt is found on the first two lines,
419 prompts will be stripped from the rest of the block.
419 prompts will be stripped from the rest of the block.
420 """
420 """
421 if initial_re is None:
421 if initial_re is None:
422 initial_re = prompt_re
422 initial_re = prompt_re
423 line = ''
423 line = ''
424 while True:
424 while True:
425 line = (yield line)
425 line = (yield line)
426
426
427 # First line of cell
427 # First line of cell
428 if line is None:
428 if line is None:
429 continue
429 continue
430 out, n1 = initial_re.subn('', line, count=1)
430 out, n1 = initial_re.subn('', line, count=1)
431 line = (yield out)
431 line = (yield out)
432
432
433 if line is None:
433 if line is None:
434 continue
434 continue
435 # check for any prompt on the second line of the cell,
435 # check for any prompt on the second line of the cell,
436 # because people often copy from just after the first prompt,
436 # because people often copy from just after the first prompt,
437 # so we might not see it in the first line.
437 # so we might not see it in the first line.
438 out, n2 = prompt_re.subn('', line, count=1)
438 out, n2 = prompt_re.subn('', line, count=1)
439 line = (yield out)
439 line = (yield out)
440
440
441 if n1 or n2:
441 if n1 or n2:
442 # Found a prompt in the first two lines - check for it in
442 # Found a prompt in the first two lines - check for it in
443 # the rest of the cell as well.
443 # the rest of the cell as well.
444 while line is not None:
444 while line is not None:
445 line = (yield prompt_re.sub('', line, count=1))
445 line = (yield prompt_re.sub('', line, count=1))
446
446
447 else:
447 else:
448 # Prompts not in input - wait for reset
448 # Prompts not in input - wait for reset
449 while line is not None:
449 while line is not None:
450 line = (yield line)
450 line = (yield line)
451
451
452 @CoroutineInputTransformer.wrap
452 @CoroutineInputTransformer.wrap
453 def classic_prompt():
453 def classic_prompt():
454 """Strip the >>>/... prompts of the Python interactive shell."""
454 """Strip the >>>/... prompts of the Python interactive shell."""
455 # FIXME: non-capturing version (?:...) usable?
455 # FIXME: non-capturing version (?:...) usable?
456 prompt_re = re.compile(r'^(>>> ?|\.\.\. ?)')
456 prompt_re = re.compile(r'^(>>> ?|\.\.\. ?)')
457 initial_re = re.compile(r'^(>>> ?)')
457 initial_re = re.compile(r'^(>>> ?)')
458 return _strip_prompts(prompt_re, initial_re)
458 return _strip_prompts(prompt_re, initial_re)
459
459
460 @CoroutineInputTransformer.wrap
460 @CoroutineInputTransformer.wrap
461 def ipy_prompt():
461 def ipy_prompt():
462 """Strip IPython's In [1]:/...: prompts."""
462 """Strip IPython's In [1]:/...: prompts."""
463 # FIXME: non-capturing version (?:...) usable?
463 # FIXME: non-capturing version (?:...) usable?
464 prompt_re = re.compile(r'^(In \[\d+\]: |\ {3,}\.{3,}: )')
464 prompt_re = re.compile(r'^(In \[\d+\]: |\ {3,}\.{3,}: )')
465 return _strip_prompts(prompt_re)
465 return _strip_prompts(prompt_re)
466
466
467
467
468 @CoroutineInputTransformer.wrap
468 @CoroutineInputTransformer.wrap
469 def leading_indent():
469 def leading_indent():
470 """Remove leading indentation.
470 """Remove leading indentation.
471
471
472 If the first line starts with a spaces or tabs, the same whitespace will be
472 If the first line starts with a spaces or tabs, the same whitespace will be
473 removed from each following line until it is reset.
473 removed from each following line until it is reset.
474 """
474 """
475 space_re = re.compile(r'^[ \t]+')
475 space_re = re.compile(r'^[ \t]+')
476 line = ''
476 line = ''
477 while True:
477 while True:
478 line = (yield line)
478 line = (yield line)
479
479
480 if line is None:
480 if line is None:
481 continue
481 continue
482
482
483 m = space_re.match(line)
483 m = space_re.match(line)
484 if m:
484 if m:
485 space = m.group(0)
485 space = m.group(0)
486 while line is not None:
486 while line is not None:
487 if line.startswith(space):
487 if line.startswith(space):
488 line = line[len(space):]
488 line = line[len(space):]
489 line = (yield line)
489 line = (yield line)
490 else:
490 else:
491 # No leading spaces - wait for reset
491 # No leading spaces - wait for reset
492 while line is not None:
492 while line is not None:
493 line = (yield line)
493 line = (yield line)
494
494
495
495
496 @CoroutineInputTransformer.wrap
496 @CoroutineInputTransformer.wrap
497 def strip_encoding_cookie():
497 def strip_encoding_cookie():
498 """Remove encoding comment if found in first two lines
498 """Remove encoding comment if found in first two lines
499
499
500 If the first or second line has the `# coding: utf-8` comment,
500 If the first or second line has the `# coding: utf-8` comment,
501 it will be removed.
501 it will be removed.
502 """
502 """
503 line = ''
503 line = ''
504 while True:
504 while True:
505 line = (yield line)
505 line = (yield line)
506 # check comment on first two lines
506 # check comment on first two lines
507 for i in range(2):
507 for i in range(2):
508 if line is None:
508 if line is None:
509 break
509 break
510 if cookie_comment_re.match(line):
510 if cookie_comment_re.match(line):
511 line = (yield "")
511 line = (yield "")
512 else:
512 else:
513 line = (yield line)
513 line = (yield line)
514
514
515 # no-op on the rest of the cell
515 # no-op on the rest of the cell
516 while line is not None:
516 while line is not None:
517 line = (yield line)
517 line = (yield line)
518
518
519 _assign_pat = \
520 r'''(?P<lhs>(\s*)
521 ([\w\.]+) # Initial identifier
522 (\s*,\s*
523 \*?[\w\.]+)* # Further identifiers for unpacking
524 \s*?,? # Trailing comma
525 )
526 \s*=\s*
527 '''
519
528
520 assign_system_re = re.compile(r'(?P<lhs>(\s*)([\w\.]+)((\s*,\s*[\w\.]+)*))'
529 assign_system_re = re.compile(r'{}!\s*(?P<cmd>.*)'.format(_assign_pat), re.VERBOSE)
521 r'\s*=\s*!\s*(?P<cmd>.*)')
522 assign_system_template = '%s = get_ipython().getoutput(%r)'
530 assign_system_template = '%s = get_ipython().getoutput(%r)'
523 @StatelessInputTransformer.wrap
531 @StatelessInputTransformer.wrap
524 def assign_from_system(line):
532 def assign_from_system(line):
525 """Transform assignment from system commands (e.g. files = !ls)"""
533 """Transform assignment from system commands (e.g. files = !ls)"""
526 m = assign_system_re.match(line)
534 m = assign_system_re.match(line)
527 if m is None:
535 if m is None:
528 return line
536 return line
529
537
530 return assign_system_template % m.group('lhs', 'cmd')
538 return assign_system_template % m.group('lhs', 'cmd')
531
539
532 assign_magic_re = re.compile(r'(?P<lhs>(\s*)([\w\.]+)((\s*,\s*[\w\.]+)*))'
540 assign_magic_re = re.compile(r'{}%\s*(?P<cmd>.*)'.format(_assign_pat), re.VERBOSE)
533 r'\s*=\s*%\s*(?P<cmd>.*)')
534 assign_magic_template = '%s = get_ipython().magic(%r)'
541 assign_magic_template = '%s = get_ipython().magic(%r)'
535 @StatelessInputTransformer.wrap
542 @StatelessInputTransformer.wrap
536 def assign_from_magic(line):
543 def assign_from_magic(line):
537 """Transform assignment from magic commands (e.g. a = %who_ls)"""
544 """Transform assignment from magic commands (e.g. a = %who_ls)"""
538 m = assign_magic_re.match(line)
545 m = assign_magic_re.match(line)
539 if m is None:
546 if m is None:
540 return line
547 return line
541
548
542 return assign_magic_template % m.group('lhs', 'cmd')
549 return assign_magic_template % m.group('lhs', 'cmd')
General Comments 0
You need to be logged in to leave comments. Login now