##// END OF EJS Templates
Improve assignment regex to match more tuple unpacking syntax...
Thomas Kluyver -
Show More
@@ -1,542 +1,549 b''
1 1 """Input transformer classes to support IPython special syntax.
2 2
3 3 This includes the machinery to recognise and transform ``%magic`` commands,
4 4 ``!system`` commands, ``help?`` querying, prompt stripping, and so forth.
5 5 """
6 6 import abc
7 7 import functools
8 8 import re
9 9
10 10 from IPython.core.splitinput import LineInfo
11 11 from IPython.utils import tokenize2
12 12 from IPython.utils.openpy import cookie_comment_re
13 13 from IPython.utils.py3compat import with_metaclass, PY3
14 14 from IPython.utils.tokenize2 import generate_tokens, untokenize, TokenError
15 15
16 16 if PY3:
17 17 from io import StringIO
18 18 else:
19 19 from StringIO import StringIO
20 20
21 21 #-----------------------------------------------------------------------------
22 22 # Globals
23 23 #-----------------------------------------------------------------------------
24 24
25 25 # The escape sequences that define the syntax transformations IPython will
26 26 # apply to user input. These can NOT be just changed here: many regular
27 27 # expressions and other parts of the code may use their hardcoded values, and
28 28 # for all intents and purposes they constitute the 'IPython syntax', so they
29 29 # should be considered fixed.
30 30
31 31 ESC_SHELL = '!' # Send line to underlying system shell
32 32 ESC_SH_CAP = '!!' # Send line to system shell and capture output
33 33 ESC_HELP = '?' # Find information about object
34 34 ESC_HELP2 = '??' # Find extra-detailed information about object
35 35 ESC_MAGIC = '%' # Call magic function
36 36 ESC_MAGIC2 = '%%' # Call cell-magic function
37 37 ESC_QUOTE = ',' # Split args on whitespace, quote each as string and call
38 38 ESC_QUOTE2 = ';' # Quote all args as a single string, call
39 39 ESC_PAREN = '/' # Call first argument with rest of line as arguments
40 40
41 41 ESC_SEQUENCES = [ESC_SHELL, ESC_SH_CAP, ESC_HELP ,\
42 42 ESC_HELP2, ESC_MAGIC, ESC_MAGIC2,\
43 43 ESC_QUOTE, ESC_QUOTE2, ESC_PAREN ]
44 44
45 45
46 46 class InputTransformer(with_metaclass(abc.ABCMeta, object)):
47 47 """Abstract base class for line-based input transformers."""
48 48
49 49 @abc.abstractmethod
50 50 def push(self, line):
51 51 """Send a line of input to the transformer, returning the transformed
52 52 input or None if the transformer is waiting for more input.
53 53
54 54 Must be overridden by subclasses.
55 55
56 56 Implementations may raise ``SyntaxError`` if the input is invalid. No
57 57 other exceptions may be raised.
58 58 """
59 59 pass
60 60
61 61 @abc.abstractmethod
62 62 def reset(self):
63 63 """Return, transformed any lines that the transformer has accumulated,
64 64 and reset its internal state.
65 65
66 66 Must be overridden by subclasses.
67 67 """
68 68 pass
69 69
70 70 @classmethod
71 71 def wrap(cls, func):
72 72 """Can be used by subclasses as a decorator, to return a factory that
73 73 will allow instantiation with the decorated object.
74 74 """
75 75 @functools.wraps(func)
76 76 def transformer_factory(**kwargs):
77 77 return cls(func, **kwargs)
78 78
79 79 return transformer_factory
80 80
81 81 class StatelessInputTransformer(InputTransformer):
82 82 """Wrapper for a stateless input transformer implemented as a function."""
83 83 def __init__(self, func):
84 84 self.func = func
85 85
86 86 def __repr__(self):
87 87 return "StatelessInputTransformer(func={0!r})".format(self.func)
88 88
89 89 def push(self, line):
90 90 """Send a line of input to the transformer, returning the
91 91 transformed input."""
92 92 return self.func(line)
93 93
94 94 def reset(self):
95 95 """No-op - exists for compatibility."""
96 96 pass
97 97
98 98 class CoroutineInputTransformer(InputTransformer):
99 99 """Wrapper for an input transformer implemented as a coroutine."""
100 100 def __init__(self, coro, **kwargs):
101 101 # Prime it
102 102 self.coro = coro(**kwargs)
103 103 next(self.coro)
104 104
105 105 def __repr__(self):
106 106 return "CoroutineInputTransformer(coro={0!r})".format(self.coro)
107 107
108 108 def push(self, line):
109 109 """Send a line of input to the transformer, returning the
110 110 transformed input or None if the transformer is waiting for more
111 111 input.
112 112 """
113 113 return self.coro.send(line)
114 114
115 115 def reset(self):
116 116 """Return, transformed any lines that the transformer has
117 117 accumulated, and reset its internal state.
118 118 """
119 119 return self.coro.send(None)
120 120
121 121 class TokenInputTransformer(InputTransformer):
122 122 """Wrapper for a token-based input transformer.
123 123
124 124 func should accept a list of tokens (5-tuples, see tokenize docs), and
125 125 return an iterable which can be passed to tokenize.untokenize().
126 126 """
127 127 def __init__(self, func):
128 128 self.func = func
129 129 self.current_line = ""
130 130 self.line_used = False
131 131 self.reset_tokenizer()
132 132
133 133 def reset_tokenizer(self):
134 134 self.tokenizer = generate_tokens(self.get_line)
135 135
136 136 def get_line(self):
137 137 if self.line_used:
138 138 raise TokenError
139 139 self.line_used = True
140 140 return self.current_line
141 141
142 142 def push(self, line):
143 143 self.current_line += line + "\n"
144 144 if self.current_line.isspace():
145 145 return self.reset()
146 146
147 147 self.line_used = False
148 148 tokens = []
149 149 stop_at_NL = False
150 150 try:
151 151 for intok in self.tokenizer:
152 152 tokens.append(intok)
153 153 t = intok[0]
154 154 if t == tokenize2.NEWLINE or (stop_at_NL and t == tokenize2.NL):
155 155 # Stop before we try to pull a line we don't have yet
156 156 break
157 157 elif t == tokenize2.ERRORTOKEN:
158 158 stop_at_NL = True
159 159 except TokenError:
160 160 # Multi-line statement - stop and try again with the next line
161 161 self.reset_tokenizer()
162 162 return None
163 163
164 164 return self.output(tokens)
165 165
166 166 def output(self, tokens):
167 167 self.current_line = ""
168 168 self.reset_tokenizer()
169 169 return untokenize(self.func(tokens)).rstrip('\n')
170 170
171 171 def reset(self):
172 172 l = self.current_line
173 173 self.current_line = ""
174 174 self.reset_tokenizer()
175 175 if l:
176 176 return l.rstrip('\n')
177 177
178 178 class assemble_python_lines(TokenInputTransformer):
179 179 def __init__(self):
180 180 super(assemble_python_lines, self).__init__(None)
181 181
182 182 def output(self, tokens):
183 183 return self.reset()
184 184
185 185 @CoroutineInputTransformer.wrap
186 186 def assemble_logical_lines():
187 187 """Join lines following explicit line continuations (\)"""
188 188 line = ''
189 189 while True:
190 190 line = (yield line)
191 191 if not line or line.isspace():
192 192 continue
193 193
194 194 parts = []
195 195 while line is not None:
196 196 if line.endswith('\\') and (not has_comment(line)):
197 197 parts.append(line[:-1])
198 198 line = (yield None) # Get another line
199 199 else:
200 200 parts.append(line)
201 201 break
202 202
203 203 # Output
204 204 line = ''.join(parts)
205 205
206 206 # Utilities
207 207 def _make_help_call(target, esc, lspace, next_input=None):
208 208 """Prepares a pinfo(2)/psearch call from a target name and the escape
209 209 (i.e. ? or ??)"""
210 210 method = 'pinfo2' if esc == '??' \
211 211 else 'psearch' if '*' in target \
212 212 else 'pinfo'
213 213 arg = " ".join([method, target])
214 214 if next_input is None:
215 215 return '%sget_ipython().magic(%r)' % (lspace, arg)
216 216 else:
217 217 return '%sget_ipython().set_next_input(%r);get_ipython().magic(%r)' % \
218 218 (lspace, next_input, arg)
219 219
220 220 # These define the transformations for the different escape characters.
221 221 def _tr_system(line_info):
222 222 "Translate lines escaped with: !"
223 223 cmd = line_info.line.lstrip().lstrip(ESC_SHELL)
224 224 return '%sget_ipython().system(%r)' % (line_info.pre, cmd)
225 225
226 226 def _tr_system2(line_info):
227 227 "Translate lines escaped with: !!"
228 228 cmd = line_info.line.lstrip()[2:]
229 229 return '%sget_ipython().getoutput(%r)' % (line_info.pre, cmd)
230 230
231 231 def _tr_help(line_info):
232 232 "Translate lines escaped with: ?/??"
233 233 # A naked help line should just fire the intro help screen
234 234 if not line_info.line[1:]:
235 235 return 'get_ipython().show_usage()'
236 236
237 237 return _make_help_call(line_info.ifun, line_info.esc, line_info.pre)
238 238
239 239 def _tr_magic(line_info):
240 240 "Translate lines escaped with: %"
241 241 tpl = '%sget_ipython().magic(%r)'
242 242 if line_info.line.startswith(ESC_MAGIC2):
243 243 return line_info.line
244 244 cmd = ' '.join([line_info.ifun, line_info.the_rest]).strip()
245 245 return tpl % (line_info.pre, cmd)
246 246
247 247 def _tr_quote(line_info):
248 248 "Translate lines escaped with: ,"
249 249 return '%s%s("%s")' % (line_info.pre, line_info.ifun,
250 250 '", "'.join(line_info.the_rest.split()) )
251 251
252 252 def _tr_quote2(line_info):
253 253 "Translate lines escaped with: ;"
254 254 return '%s%s("%s")' % (line_info.pre, line_info.ifun,
255 255 line_info.the_rest)
256 256
257 257 def _tr_paren(line_info):
258 258 "Translate lines escaped with: /"
259 259 return '%s%s(%s)' % (line_info.pre, line_info.ifun,
260 260 ", ".join(line_info.the_rest.split()))
261 261
262 262 tr = { ESC_SHELL : _tr_system,
263 263 ESC_SH_CAP : _tr_system2,
264 264 ESC_HELP : _tr_help,
265 265 ESC_HELP2 : _tr_help,
266 266 ESC_MAGIC : _tr_magic,
267 267 ESC_QUOTE : _tr_quote,
268 268 ESC_QUOTE2 : _tr_quote2,
269 269 ESC_PAREN : _tr_paren }
270 270
271 271 @StatelessInputTransformer.wrap
272 272 def escaped_commands(line):
273 273 """Transform escaped commands - %magic, !system, ?help + various autocalls.
274 274 """
275 275 if not line or line.isspace():
276 276 return line
277 277 lineinf = LineInfo(line)
278 278 if lineinf.esc not in tr:
279 279 return line
280 280
281 281 return tr[lineinf.esc](lineinf)
282 282
283 283 _initial_space_re = re.compile(r'\s*')
284 284
285 285 _help_end_re = re.compile(r"""(%{0,2}
286 286 [a-zA-Z_*][\w*]* # Variable name
287 287 (\.[a-zA-Z_*][\w*]*)* # .etc.etc
288 288 )
289 289 (\?\??)$ # ? or ??
290 290 """,
291 291 re.VERBOSE)
292 292
293 293 # Extra pseudotokens for multiline strings and data structures
294 294 _MULTILINE_STRING = object()
295 295 _MULTILINE_STRUCTURE = object()
296 296
297 297 def _line_tokens(line):
298 298 """Helper for has_comment and ends_in_comment_or_string."""
299 299 readline = StringIO(line).readline
300 300 toktypes = set()
301 301 try:
302 302 for t in generate_tokens(readline):
303 303 toktypes.add(t[0])
304 304 except TokenError as e:
305 305 # There are only two cases where a TokenError is raised.
306 306 if 'multi-line string' in e.args[0]:
307 307 toktypes.add(_MULTILINE_STRING)
308 308 else:
309 309 toktypes.add(_MULTILINE_STRUCTURE)
310 310 return toktypes
311 311
312 312 def has_comment(src):
313 313 """Indicate whether an input line has (i.e. ends in, or is) a comment.
314 314
315 315 This uses tokenize, so it can distinguish comments from # inside strings.
316 316
317 317 Parameters
318 318 ----------
319 319 src : string
320 320 A single line input string.
321 321
322 322 Returns
323 323 -------
324 324 comment : bool
325 325 True if source has a comment.
326 326 """
327 327 return (tokenize2.COMMENT in _line_tokens(src))
328 328
329 329 def ends_in_comment_or_string(src):
330 330 """Indicates whether or not an input line ends in a comment or within
331 331 a multiline string.
332 332
333 333 Parameters
334 334 ----------
335 335 src : string
336 336 A single line input string.
337 337
338 338 Returns
339 339 -------
340 340 comment : bool
341 341 True if source ends in a comment or multiline string.
342 342 """
343 343 toktypes = _line_tokens(src)
344 344 return (tokenize2.COMMENT in toktypes) or (_MULTILINE_STRING in toktypes)
345 345
346 346
347 347 @StatelessInputTransformer.wrap
348 348 def help_end(line):
349 349 """Translate lines with ?/?? at the end"""
350 350 m = _help_end_re.search(line)
351 351 if m is None or ends_in_comment_or_string(line):
352 352 return line
353 353 target = m.group(1)
354 354 esc = m.group(3)
355 355 lspace = _initial_space_re.match(line).group(0)
356 356
357 357 # If we're mid-command, put it back on the next prompt for the user.
358 358 next_input = line.rstrip('?') if line.strip() != m.group(0) else None
359 359
360 360 return _make_help_call(target, esc, lspace, next_input)
361 361
362 362
363 363 @CoroutineInputTransformer.wrap
364 364 def cellmagic(end_on_blank_line=False):
365 365 """Captures & transforms cell magics.
366 366
367 367 After a cell magic is started, this stores up any lines it gets until it is
368 368 reset (sent None).
369 369 """
370 370 tpl = 'get_ipython().run_cell_magic(%r, %r, %r)'
371 371 cellmagic_help_re = re.compile('%%\w+\?')
372 372 line = ''
373 373 while True:
374 374 line = (yield line)
375 375 # consume leading empty lines
376 376 while not line:
377 377 line = (yield line)
378 378
379 379 if not line.startswith(ESC_MAGIC2):
380 380 # This isn't a cell magic, idle waiting for reset then start over
381 381 while line is not None:
382 382 line = (yield line)
383 383 continue
384 384
385 385 if cellmagic_help_re.match(line):
386 386 # This case will be handled by help_end
387 387 continue
388 388
389 389 first = line
390 390 body = []
391 391 line = (yield None)
392 392 while (line is not None) and \
393 393 ((line.strip() != '') or not end_on_blank_line):
394 394 body.append(line)
395 395 line = (yield None)
396 396
397 397 # Output
398 398 magic_name, _, first = first.partition(' ')
399 399 magic_name = magic_name.lstrip(ESC_MAGIC2)
400 400 line = tpl % (magic_name, first, u'\n'.join(body))
401 401
402 402
403 403 def _strip_prompts(prompt_re, initial_re=None):
404 404 """Remove matching input prompts from a block of input.
405 405
406 406 Parameters
407 407 ----------
408 408 prompt_re : regular expression
409 409 A regular expression matching any input prompt (including continuation)
410 410 initial_re : regular expression, optional
411 411 A regular expression matching only the initial prompt, but not continuation.
412 412 If no initial expression is given, prompt_re will be used everywhere.
413 413 Used mainly for plain Python prompts, where the continuation prompt
414 414 ``...`` is a valid Python expression in Python 3, so shouldn't be stripped.
415 415
416 416 If initial_re and prompt_re differ,
417 417 only initial_re will be tested against the first line.
418 418 If any prompt is found on the first two lines,
419 419 prompts will be stripped from the rest of the block.
420 420 """
421 421 if initial_re is None:
422 422 initial_re = prompt_re
423 423 line = ''
424 424 while True:
425 425 line = (yield line)
426 426
427 427 # First line of cell
428 428 if line is None:
429 429 continue
430 430 out, n1 = initial_re.subn('', line, count=1)
431 431 line = (yield out)
432 432
433 433 if line is None:
434 434 continue
435 435 # check for any prompt on the second line of the cell,
436 436 # because people often copy from just after the first prompt,
437 437 # so we might not see it in the first line.
438 438 out, n2 = prompt_re.subn('', line, count=1)
439 439 line = (yield out)
440 440
441 441 if n1 or n2:
442 442 # Found a prompt in the first two lines - check for it in
443 443 # the rest of the cell as well.
444 444 while line is not None:
445 445 line = (yield prompt_re.sub('', line, count=1))
446 446
447 447 else:
448 448 # Prompts not in input - wait for reset
449 449 while line is not None:
450 450 line = (yield line)
451 451
452 452 @CoroutineInputTransformer.wrap
453 453 def classic_prompt():
454 454 """Strip the >>>/... prompts of the Python interactive shell."""
455 455 # FIXME: non-capturing version (?:...) usable?
456 456 prompt_re = re.compile(r'^(>>> ?|\.\.\. ?)')
457 457 initial_re = re.compile(r'^(>>> ?)')
458 458 return _strip_prompts(prompt_re, initial_re)
459 459
460 460 @CoroutineInputTransformer.wrap
461 461 def ipy_prompt():
462 462 """Strip IPython's In [1]:/...: prompts."""
463 463 # FIXME: non-capturing version (?:...) usable?
464 464 prompt_re = re.compile(r'^(In \[\d+\]: |\ {3,}\.{3,}: )')
465 465 return _strip_prompts(prompt_re)
466 466
467 467
468 468 @CoroutineInputTransformer.wrap
469 469 def leading_indent():
470 470 """Remove leading indentation.
471 471
472 472 If the first line starts with a spaces or tabs, the same whitespace will be
473 473 removed from each following line until it is reset.
474 474 """
475 475 space_re = re.compile(r'^[ \t]+')
476 476 line = ''
477 477 while True:
478 478 line = (yield line)
479 479
480 480 if line is None:
481 481 continue
482 482
483 483 m = space_re.match(line)
484 484 if m:
485 485 space = m.group(0)
486 486 while line is not None:
487 487 if line.startswith(space):
488 488 line = line[len(space):]
489 489 line = (yield line)
490 490 else:
491 491 # No leading spaces - wait for reset
492 492 while line is not None:
493 493 line = (yield line)
494 494
495 495
496 496 @CoroutineInputTransformer.wrap
497 497 def strip_encoding_cookie():
498 498 """Remove encoding comment if found in first two lines
499 499
500 500 If the first or second line has the `# coding: utf-8` comment,
501 501 it will be removed.
502 502 """
503 503 line = ''
504 504 while True:
505 505 line = (yield line)
506 506 # check comment on first two lines
507 507 for i in range(2):
508 508 if line is None:
509 509 break
510 510 if cookie_comment_re.match(line):
511 511 line = (yield "")
512 512 else:
513 513 line = (yield line)
514 514
515 515 # no-op on the rest of the cell
516 516 while line is not None:
517 517 line = (yield line)
518 518
519 _assign_pat = \
520 r'''(?P<lhs>(\s*)
521 ([\w\.]+) # Initial identifier
522 (\s*,\s*
523 \*?[\w\.]+)* # Further identifiers for unpacking
524 \s*?,? # Trailing comma
525 )
526 \s*=\s*
527 '''
519 528
520 assign_system_re = re.compile(r'(?P<lhs>(\s*)([\w\.]+)((\s*,\s*[\w\.]+)*))'
521 r'\s*=\s*!\s*(?P<cmd>.*)')
529 assign_system_re = re.compile(r'{}!\s*(?P<cmd>.*)'.format(_assign_pat), re.VERBOSE)
522 530 assign_system_template = '%s = get_ipython().getoutput(%r)'
523 531 @StatelessInputTransformer.wrap
524 532 def assign_from_system(line):
525 533 """Transform assignment from system commands (e.g. files = !ls)"""
526 534 m = assign_system_re.match(line)
527 535 if m is None:
528 536 return line
529 537
530 538 return assign_system_template % m.group('lhs', 'cmd')
531 539
532 assign_magic_re = re.compile(r'(?P<lhs>(\s*)([\w\.]+)((\s*,\s*[\w\.]+)*))'
533 r'\s*=\s*%\s*(?P<cmd>.*)')
540 assign_magic_re = re.compile(r'{}%\s*(?P<cmd>.*)'.format(_assign_pat), re.VERBOSE)
534 541 assign_magic_template = '%s = get_ipython().magic(%r)'
535 542 @StatelessInputTransformer.wrap
536 543 def assign_from_magic(line):
537 544 """Transform assignment from magic commands (e.g. a = %who_ls)"""
538 545 m = assign_magic_re.match(line)
539 546 if m is None:
540 547 return line
541 548
542 549 return assign_magic_template % m.group('lhs', 'cmd')
General Comments 0
You need to be logged in to leave comments. Login now