##// END OF EJS Templates
format positions are required in Python 2.6.x
Jason Grout -
Show More
@@ -1,444 +1,444 b''
1 1 import abc
2 2 import functools
3 3 import re
4 4 from StringIO import StringIO
5 5
6 6 from IPython.core.splitinput import LineInfo
7 7 from IPython.utils import tokenize2
8 8 from IPython.utils.tokenize2 import generate_tokens, untokenize, TokenError
9 9
10 10 #-----------------------------------------------------------------------------
11 11 # Globals
12 12 #-----------------------------------------------------------------------------
13 13
14 14 # The escape sequences that define the syntax transformations IPython will
15 15 # apply to user input. These can NOT be just changed here: many regular
16 16 # expressions and other parts of the code may use their hardcoded values, and
17 17 # for all intents and purposes they constitute the 'IPython syntax', so they
18 18 # should be considered fixed.
19 19
20 20 ESC_SHELL = '!' # Send line to underlying system shell
21 21 ESC_SH_CAP = '!!' # Send line to system shell and capture output
22 22 ESC_HELP = '?' # Find information about object
23 23 ESC_HELP2 = '??' # Find extra-detailed information about object
24 24 ESC_MAGIC = '%' # Call magic function
25 25 ESC_MAGIC2 = '%%' # Call cell-magic function
26 26 ESC_QUOTE = ',' # Split args on whitespace, quote each as string and call
27 27 ESC_QUOTE2 = ';' # Quote all args as a single string, call
28 28 ESC_PAREN = '/' # Call first argument with rest of line as arguments
29 29
30 30 ESC_SEQUENCES = [ESC_SHELL, ESC_SH_CAP, ESC_HELP ,\
31 31 ESC_HELP2, ESC_MAGIC, ESC_MAGIC2,\
32 32 ESC_QUOTE, ESC_QUOTE2, ESC_PAREN ]
33 33
34 34
35 35 class InputTransformer(object):
36 36 """Abstract base class for line-based input transformers."""
37 37 __metaclass__ = abc.ABCMeta
38 38
39 39 @abc.abstractmethod
40 40 def push(self, line):
41 41 """Send a line of input to the transformer, returning the transformed
42 42 input or None if the transformer is waiting for more input.
43 43
44 44 Must be overridden by subclasses.
45 45 """
46 46 pass
47 47
48 48 @abc.abstractmethod
49 49 def reset(self):
50 50 """Return, transformed any lines that the transformer has accumulated,
51 51 and reset its internal state.
52 52
53 53 Must be overridden by subclasses.
54 54 """
55 55 pass
56 56
57 57 @classmethod
58 58 def wrap(cls, func):
59 59 """Can be used by subclasses as a decorator, to return a factory that
60 60 will allow instantiation with the decorated object.
61 61 """
62 62 @functools.wraps(func)
63 63 def transformer_factory(**kwargs):
64 64 return cls(func, **kwargs)
65 65
66 66 return transformer_factory
67 67
68 68 class StatelessInputTransformer(InputTransformer):
69 69 """Wrapper for a stateless input transformer implemented as a function."""
70 70 def __init__(self, func):
71 71 self.func = func
72 72
73 73 def __repr__(self):
74 return "StatelessInputTransformer(func={!r})".format(self.func)
74 return "StatelessInputTransformer(func={0!r})".format(self.func)
75 75
76 76 def push(self, line):
77 77 """Send a line of input to the transformer, returning the
78 78 transformed input."""
79 79 return self.func(line)
80 80
81 81 def reset(self):
82 82 """No-op - exists for compatibility."""
83 83 pass
84 84
85 85 class CoroutineInputTransformer(InputTransformer):
86 86 """Wrapper for an input transformer implemented as a coroutine."""
87 87 def __init__(self, coro, **kwargs):
88 88 # Prime it
89 89 self.coro = coro(**kwargs)
90 90 next(self.coro)
91 91
92 92 def __repr__(self):
93 return "CoroutineInputTransformer(coro={!r})".format(self.coro)
93 return "CoroutineInputTransformer(coro={0!r})".format(self.coro)
94 94
95 95 def push(self, line):
96 96 """Send a line of input to the transformer, returning the
97 97 transformed input or None if the transformer is waiting for more
98 98 input.
99 99 """
100 100 return self.coro.send(line)
101 101
102 102 def reset(self):
103 103 """Return, transformed any lines that the transformer has
104 104 accumulated, and reset its internal state.
105 105 """
106 106 return self.coro.send(None)
107 107
108 108 class TokenInputTransformer(InputTransformer):
109 109 """Wrapper for a token-based input transformer.
110 110
111 111 func should accept a list of tokens (5-tuples, see tokenize docs), and
112 112 return an iterable which can be passed to tokenize.untokenize().
113 113 """
114 114 def __init__(self, func):
115 115 self.func = func
116 116 self.current_line = ""
117 117 self.line_used = False
118 118 self.reset_tokenizer()
119 119
120 120 def reset_tokenizer(self):
121 121 self.tokenizer = generate_tokens(self.get_line)
122 122
123 123 def get_line(self):
124 124 if self.line_used:
125 125 raise TokenError
126 126 self.line_used = True
127 127 return self.current_line
128 128
129 129 def push(self, line):
130 130 self.current_line += line + "\n"
131 131 if self.current_line.isspace():
132 132 return self.reset()
133 133
134 134 self.line_used = False
135 135 tokens = []
136 136 stop_at_NL = False
137 137 try:
138 138 for intok in self.tokenizer:
139 139 tokens.append(intok)
140 140 t = intok[0]
141 141 if t == tokenize2.NEWLINE or (stop_at_NL and t == tokenize2.NL):
142 142 # Stop before we try to pull a line we don't have yet
143 143 break
144 144 elif t == tokenize2.ERRORTOKEN:
145 145 stop_at_NL = True
146 146 except TokenError:
147 147 # Multi-line statement - stop and try again with the next line
148 148 self.reset_tokenizer()
149 149 return None
150 150
151 151 return self.output(tokens)
152 152
153 153 def output(self, tokens):
154 154 self.current_line = ""
155 155 self.reset_tokenizer()
156 156 return untokenize(self.func(tokens)).rstrip('\n')
157 157
158 158 def reset(self):
159 159 l = self.current_line
160 160 self.current_line = ""
161 161 self.reset_tokenizer()
162 162 if l:
163 163 return l.rstrip('\n')
164 164
165 165 class assemble_python_lines(TokenInputTransformer):
166 166 def __init__(self):
167 167 super(assemble_python_lines, self).__init__(None)
168 168
169 169 def output(self, tokens):
170 170 return self.reset()
171 171
172 172 @CoroutineInputTransformer.wrap
173 173 def assemble_logical_lines():
174 174 """Join lines following explicit line continuations (\)"""
175 175 line = ''
176 176 while True:
177 177 line = (yield line)
178 178 if not line or line.isspace():
179 179 continue
180 180
181 181 parts = []
182 182 while line is not None:
183 183 if line.endswith('\\') and (not has_comment(line)):
184 184 parts.append(line[:-1])
185 185 line = (yield None) # Get another line
186 186 else:
187 187 parts.append(line)
188 188 break
189 189
190 190 # Output
191 191 line = ''.join(parts)
192 192
193 193 # Utilities
194 194 def _make_help_call(target, esc, lspace, next_input=None):
195 195 """Prepares a pinfo(2)/psearch call from a target name and the escape
196 196 (i.e. ? or ??)"""
197 197 method = 'pinfo2' if esc == '??' \
198 198 else 'psearch' if '*' in target \
199 199 else 'pinfo'
200 200 arg = " ".join([method, target])
201 201 if next_input is None:
202 202 return '%sget_ipython().magic(%r)' % (lspace, arg)
203 203 else:
204 204 return '%sget_ipython().set_next_input(%r);get_ipython().magic(%r)' % \
205 205 (lspace, next_input, arg)
206 206
207 207 # These define the transformations for the different escape characters.
208 208 def _tr_system(line_info):
209 209 "Translate lines escaped with: !"
210 210 cmd = line_info.line.lstrip().lstrip(ESC_SHELL)
211 211 return '%sget_ipython().system(%r)' % (line_info.pre, cmd)
212 212
213 213 def _tr_system2(line_info):
214 214 "Translate lines escaped with: !!"
215 215 cmd = line_info.line.lstrip()[2:]
216 216 return '%sget_ipython().getoutput(%r)' % (line_info.pre, cmd)
217 217
218 218 def _tr_help(line_info):
219 219 "Translate lines escaped with: ?/??"
220 220 # A naked help line should just fire the intro help screen
221 221 if not line_info.line[1:]:
222 222 return 'get_ipython().show_usage()'
223 223
224 224 return _make_help_call(line_info.ifun, line_info.esc, line_info.pre)
225 225
226 226 def _tr_magic(line_info):
227 227 "Translate lines escaped with: %"
228 228 tpl = '%sget_ipython().magic(%r)'
229 229 cmd = ' '.join([line_info.ifun, line_info.the_rest]).strip()
230 230 return tpl % (line_info.pre, cmd)
231 231
232 232 def _tr_quote(line_info):
233 233 "Translate lines escaped with: ,"
234 234 return '%s%s("%s")' % (line_info.pre, line_info.ifun,
235 235 '", "'.join(line_info.the_rest.split()) )
236 236
237 237 def _tr_quote2(line_info):
238 238 "Translate lines escaped with: ;"
239 239 return '%s%s("%s")' % (line_info.pre, line_info.ifun,
240 240 line_info.the_rest)
241 241
242 242 def _tr_paren(line_info):
243 243 "Translate lines escaped with: /"
244 244 return '%s%s(%s)' % (line_info.pre, line_info.ifun,
245 245 ", ".join(line_info.the_rest.split()))
246 246
247 247 tr = { ESC_SHELL : _tr_system,
248 248 ESC_SH_CAP : _tr_system2,
249 249 ESC_HELP : _tr_help,
250 250 ESC_HELP2 : _tr_help,
251 251 ESC_MAGIC : _tr_magic,
252 252 ESC_QUOTE : _tr_quote,
253 253 ESC_QUOTE2 : _tr_quote2,
254 254 ESC_PAREN : _tr_paren }
255 255
256 256 @StatelessInputTransformer.wrap
257 257 def escaped_commands(line):
258 258 """Transform escaped commands - %magic, !system, ?help + various autocalls.
259 259 """
260 260 if not line or line.isspace():
261 261 return line
262 262 lineinf = LineInfo(line)
263 263 if lineinf.esc not in tr:
264 264 return line
265 265
266 266 return tr[lineinf.esc](lineinf)
267 267
268 268 _initial_space_re = re.compile(r'\s*')
269 269
270 270 _help_end_re = re.compile(r"""(%{0,2}
271 271 [a-zA-Z_*][\w*]* # Variable name
272 272 (\.[a-zA-Z_*][\w*]*)* # .etc.etc
273 273 )
274 274 (\?\??)$ # ? or ??""",
275 275 re.VERBOSE)
276 276
277 277 def has_comment(src):
278 278 """Indicate whether an input line has (i.e. ends in, or is) a comment.
279 279
280 280 This uses tokenize, so it can distinguish comments from # inside strings.
281 281
282 282 Parameters
283 283 ----------
284 284 src : string
285 285 A single line input string.
286 286
287 287 Returns
288 288 -------
289 289 comment : bool
290 290 True if source has a comment.
291 291 """
292 292 readline = StringIO(src).readline
293 293 toktypes = set()
294 294 try:
295 295 for t in generate_tokens(readline):
296 296 toktypes.add(t[0])
297 297 except TokenError:
298 298 pass
299 299 return(tokenize2.COMMENT in toktypes)
300 300
301 301
302 302 @StatelessInputTransformer.wrap
303 303 def help_end(line):
304 304 """Translate lines with ?/?? at the end"""
305 305 m = _help_end_re.search(line)
306 306 if m is None or has_comment(line):
307 307 return line
308 308 target = m.group(1)
309 309 esc = m.group(3)
310 310 lspace = _initial_space_re.match(line).group(0)
311 311
312 312 # If we're mid-command, put it back on the next prompt for the user.
313 313 next_input = line.rstrip('?') if line.strip() != m.group(0) else None
314 314
315 315 return _make_help_call(target, esc, lspace, next_input)
316 316
317 317
318 318 @CoroutineInputTransformer.wrap
319 319 def cellmagic(end_on_blank_line=False):
320 320 """Captures & transforms cell magics.
321 321
322 322 After a cell magic is started, this stores up any lines it gets until it is
323 323 reset (sent None).
324 324 """
325 325 tpl = 'get_ipython().run_cell_magic(%r, %r, %r)'
326 326 cellmagic_help_re = re.compile('%%\w+\?')
327 327 line = ''
328 328 while True:
329 329 line = (yield line)
330 330 if (not line) or (not line.startswith(ESC_MAGIC2)):
331 331 continue
332 332
333 333 if cellmagic_help_re.match(line):
334 334 # This case will be handled by help_end
335 335 continue
336 336
337 337 first = line
338 338 body = []
339 339 line = (yield None)
340 340 while (line is not None) and \
341 341 ((line.strip() != '') or not end_on_blank_line):
342 342 body.append(line)
343 343 line = (yield None)
344 344
345 345 # Output
346 346 magic_name, _, first = first.partition(' ')
347 347 magic_name = magic_name.lstrip(ESC_MAGIC2)
348 348 line = tpl % (magic_name, first, u'\n'.join(body))
349 349
350 350
351 351 def _strip_prompts(prompt_re):
352 352 """Remove matching input prompts from a block of input."""
353 353 line = ''
354 354 while True:
355 355 line = (yield line)
356 356
357 357 # First line of cell
358 358 if line is None:
359 359 continue
360 360 out, n1 = prompt_re.subn('', line, count=1)
361 361 line = (yield out)
362 362
363 363 # Second line of cell, because people often copy from just after the
364 364 # first prompt, so we might not see it in the first line.
365 365 if line is None:
366 366 continue
367 367 out, n2 = prompt_re.subn('', line, count=1)
368 368 line = (yield out)
369 369
370 370 if n1 or n2:
371 371 # Found the input prompt in the first two lines - check for it in
372 372 # the rest of the cell as well.
373 373 while line is not None:
374 374 line = (yield prompt_re.sub('', line, count=1))
375 375
376 376 else:
377 377 # Prompts not in input - wait for reset
378 378 while line is not None:
379 379 line = (yield line)
380 380
381 381 @CoroutineInputTransformer.wrap
382 382 def classic_prompt():
383 383 """Strip the >>>/... prompts of the Python interactive shell."""
384 384 prompt_re = re.compile(r'^(>>> ?|^\.\.\. ?)')
385 385 return _strip_prompts(prompt_re)
386 386
387 387 @CoroutineInputTransformer.wrap
388 388 def ipy_prompt():
389 389 """Strip IPython's In [1]:/...: prompts."""
390 390 prompt_re = re.compile(r'^(In \[\d+\]: |^\ \ \ \.\.\.+: )')
391 391 return _strip_prompts(prompt_re)
392 392
393 393
394 394 @CoroutineInputTransformer.wrap
395 395 def leading_indent():
396 396 """Remove leading indentation.
397 397
398 398 If the first line starts with a spaces or tabs, the same whitespace will be
399 399 removed from each following line until it is reset.
400 400 """
401 401 space_re = re.compile(r'^[ \t]+')
402 402 line = ''
403 403 while True:
404 404 line = (yield line)
405 405
406 406 if line is None:
407 407 continue
408 408
409 409 m = space_re.match(line)
410 410 if m:
411 411 space = m.group(0)
412 412 while line is not None:
413 413 if line.startswith(space):
414 414 line = line[len(space):]
415 415 line = (yield line)
416 416 else:
417 417 # No leading spaces - wait for reset
418 418 while line is not None:
419 419 line = (yield line)
420 420
421 421
422 422 assign_system_re = re.compile(r'(?P<lhs>(\s*)([\w\.]+)((\s*,\s*[\w\.]+)*))'
423 423 r'\s*=\s*!\s*(?P<cmd>.*)')
424 424 assign_system_template = '%s = get_ipython().getoutput(%r)'
425 425 @StatelessInputTransformer.wrap
426 426 def assign_from_system(line):
427 427 """Transform assignment from system commands (e.g. files = !ls)"""
428 428 m = assign_system_re.match(line)
429 429 if m is None:
430 430 return line
431 431
432 432 return assign_system_template % m.group('lhs', 'cmd')
433 433
434 434 assign_magic_re = re.compile(r'(?P<lhs>(\s*)([\w\.]+)((\s*,\s*[\w\.]+)*))'
435 435 r'\s*=\s*%\s*(?P<cmd>.*)')
436 436 assign_magic_template = '%s = get_ipython().magic(%r)'
437 437 @StatelessInputTransformer.wrap
438 438 def assign_from_magic(line):
439 439 """Transform assignment from magic commands (e.g. a = %who_ls)"""
440 440 m = assign_magic_re.match(line)
441 441 if m is None:
442 442 return line
443 443
444 444 return assign_magic_template % m.group('lhs', 'cmd')
General Comments 0
You need to be logged in to leave comments. Login now