##// END OF EJS Templates
Reorganise InputTransformer decorator architecture.
Thomas Kluyver -
Show More
@@ -1,382 +1,397 b''
1 1 import abc
2 import functools
2 3 import re
3 4 from StringIO import StringIO
4 5 import tokenize
5 6
6 7 from IPython.core.splitinput import split_user_input, LineInfo
7 8
8 9 #-----------------------------------------------------------------------------
9 10 # Globals
10 11 #-----------------------------------------------------------------------------
11 12
12 13 # The escape sequences that define the syntax transformations IPython will
13 14 # apply to user input. These can NOT be just changed here: many regular
14 15 # expressions and other parts of the code may use their hardcoded values, and
15 16 # for all intents and purposes they constitute the 'IPython syntax', so they
16 17 # should be considered fixed.
17 18
18 19 ESC_SHELL = '!' # Send line to underlying system shell
19 20 ESC_SH_CAP = '!!' # Send line to system shell and capture output
20 21 ESC_HELP = '?' # Find information about object
21 22 ESC_HELP2 = '??' # Find extra-detailed information about object
22 23 ESC_MAGIC = '%' # Call magic function
23 24 ESC_MAGIC2 = '%%' # Call cell-magic function
24 25 ESC_QUOTE = ',' # Split args on whitespace, quote each as string and call
25 26 ESC_QUOTE2 = ';' # Quote all args as a single string, call
26 27 ESC_PAREN = '/' # Call first argument with rest of line as arguments
27 28
28 29 ESC_SEQUENCES = [ESC_SHELL, ESC_SH_CAP, ESC_HELP ,\
29 30 ESC_HELP2, ESC_MAGIC, ESC_MAGIC2,\
30 31 ESC_QUOTE, ESC_QUOTE2, ESC_PAREN ]
31 32
32 33
33 34 class InputTransformer(object):
34 35 """Abstract base class for line-based input transformers."""
35 36 __metaclass__ = abc.ABCMeta
36 37
37 38 @abc.abstractmethod
38 39 def push(self, line):
39 40 """Send a line of input to the transformer, returning the transformed
40 41 input or None if the transformer is waiting for more input.
41 42
42 43 Must be overridden by subclasses.
43 44 """
44 45 pass
45 46
46 47 @abc.abstractmethod
47 48 def reset(self):
48 49 """Return, transformed any lines that the transformer has accumulated,
49 50 and reset its internal state.
50 51
51 52 Must be overridden by subclasses.
52 53 """
53 54 pass
54 55
55 56 # Set this to True to allow the transformer to act on lines inside strings.
56 57 look_in_string = False
57 58
58 def stateless_input_transformer(func):
59 @classmethod
60 def wrap(cls, func):
61 """Can be used by subclasses as a decorator, to return a factory that
62 will allow instantiation with the decorated object.
63 """
64 @functools.wraps(func)
65 def transformer_factory():
66 transformer = cls(func)
67 if getattr(transformer_factory, 'look_in_string', False):
68 transformer.look_in_string = True
69 return transformer
70
71 return transformer_factory
72
59 73 class StatelessInputTransformer(InputTransformer):
60 """Decorator for a stateless input transformer implemented as a function."""
61 def __init__(self):
74 """Wrapper for a stateless input transformer implemented as a function."""
75 def __init__(self, func):
62 76 self.func = func
63 77
78 def __repr__(self):
79 return "StatelessInputTransformer(func={!r})".format(self.func)
80
64 81 def push(self, line):
65 82 """Send a line of input to the transformer, returning the
66 83 transformed input."""
67 84 return self.func(line)
68 85
69 86 def reset(self):
70 87 """No-op - exists for compatibility."""
71 88 pass
72 89
73 return StatelessInputTransformer
74
75 def coroutine_input_transformer(coro):
76 90 class CoroutineInputTransformer(InputTransformer):
77 """Wrapper for input transformers based on coroutines."""
78 def __init__(self):
91 """Wrapper for an input transformer implemented as a coroutine."""
92 def __init__(self, coro):
79 93 # Prime it
80 94 self.coro = coro()
81 95 next(self.coro)
82 96
97 def __repr__(self):
98 return "CoroutineInputTransformer(coro={!r})".format(self.coro)
99
83 100 def push(self, line):
84 101 """Send a line of input to the transformer, returning the
85 102 transformed input or None if the transformer is waiting for more
86 103 input.
87 104 """
88 105 return self.coro.send(line)
89 106
90 107 def reset(self):
91 108 """Return, transformed any lines that the transformer has
92 109 accumulated, and reset its internal state.
93 110 """
94 111 return self.coro.send(None)
95 112
96 return CoroutineInputTransformer
97
98 113
99 114 # Utilities
100 115 def _make_help_call(target, esc, lspace, next_input=None):
101 116 """Prepares a pinfo(2)/psearch call from a target name and the escape
102 117 (i.e. ? or ??)"""
103 118 method = 'pinfo2' if esc == '??' \
104 119 else 'psearch' if '*' in target \
105 120 else 'pinfo'
106 121 arg = " ".join([method, target])
107 122 if next_input is None:
108 123 return '%sget_ipython().magic(%r)' % (lspace, arg)
109 124 else:
110 125 return '%sget_ipython().set_next_input(%r);get_ipython().magic(%r)' % \
111 126 (lspace, next_input, arg)
112 127
113 @coroutine_input_transformer
128 @CoroutineInputTransformer.wrap
114 129 def escaped_transformer():
115 130 """Translate lines beginning with one of IPython's escape characters.
116 131
117 132 This is stateful to allow magic commands etc. to be continued over several
118 133 lines using explicit line continuations (\ at the end of a line).
119 134 """
120 135
121 136 # These define the transformations for the different escape characters.
122 137 def _tr_system(line_info):
123 138 "Translate lines escaped with: !"
124 139 cmd = line_info.line.lstrip().lstrip(ESC_SHELL)
125 140 return '%sget_ipython().system(%r)' % (line_info.pre, cmd)
126 141
127 142 def _tr_system2(line_info):
128 143 "Translate lines escaped with: !!"
129 144 cmd = line_info.line.lstrip()[2:]
130 145 return '%sget_ipython().getoutput(%r)' % (line_info.pre, cmd)
131 146
132 147 def _tr_help(line_info):
133 148 "Translate lines escaped with: ?/??"
134 149 # A naked help line should just fire the intro help screen
135 150 if not line_info.line[1:]:
136 151 return 'get_ipython().show_usage()'
137 152
138 153 return _make_help_call(line_info.ifun, line_info.esc, line_info.pre)
139 154
140 155 def _tr_magic(line_info):
141 156 "Translate lines escaped with: %"
142 157 tpl = '%sget_ipython().magic(%r)'
143 158 cmd = ' '.join([line_info.ifun, line_info.the_rest]).strip()
144 159 return tpl % (line_info.pre, cmd)
145 160
146 161 def _tr_quote(line_info):
147 162 "Translate lines escaped with: ,"
148 163 return '%s%s("%s")' % (line_info.pre, line_info.ifun,
149 164 '", "'.join(line_info.the_rest.split()) )
150 165
151 166 def _tr_quote2(line_info):
152 167 "Translate lines escaped with: ;"
153 168 return '%s%s("%s")' % (line_info.pre, line_info.ifun,
154 169 line_info.the_rest)
155 170
156 171 def _tr_paren(line_info):
157 172 "Translate lines escaped with: /"
158 173 return '%s%s(%s)' % (line_info.pre, line_info.ifun,
159 174 ", ".join(line_info.the_rest.split()))
160 175
161 176 tr = { ESC_SHELL : _tr_system,
162 177 ESC_SH_CAP : _tr_system2,
163 178 ESC_HELP : _tr_help,
164 179 ESC_HELP2 : _tr_help,
165 180 ESC_MAGIC : _tr_magic,
166 181 ESC_QUOTE : _tr_quote,
167 182 ESC_QUOTE2 : _tr_quote2,
168 183 ESC_PAREN : _tr_paren }
169 184
170 185 line = ''
171 186 while True:
172 187 line = (yield line)
173 188 if not line or line.isspace():
174 189 continue
175 190 lineinf = LineInfo(line)
176 191 if lineinf.esc not in tr:
177 192 continue
178 193
179 194 parts = []
180 195 while line is not None:
181 196 parts.append(line.rstrip('\\'))
182 197 if not line.endswith('\\'):
183 198 break
184 199 line = (yield None)
185 200
186 201 # Output
187 202 lineinf = LineInfo(' '.join(parts))
188 203 line = tr[lineinf.esc](lineinf)
189 204
190 205 _initial_space_re = re.compile(r'\s*')
191 206
192 207 _help_end_re = re.compile(r"""(%{0,2}
193 208 [a-zA-Z_*][\w*]* # Variable name
194 209 (\.[a-zA-Z_*][\w*]*)* # .etc.etc
195 210 )
196 211 (\?\??)$ # ? or ??""",
197 212 re.VERBOSE)
198 213
199 214 def has_comment(src):
200 215 """Indicate whether an input line has (i.e. ends in, or is) a comment.
201 216
202 217 This uses tokenize, so it can distinguish comments from # inside strings.
203 218
204 219 Parameters
205 220 ----------
206 221 src : string
207 222 A single line input string.
208 223
209 224 Returns
210 225 -------
211 226 Boolean: True if source has a comment.
212 227 """
213 228 readline = StringIO(src).readline
214 229 toktypes = set()
215 230 try:
216 231 for t in tokenize.generate_tokens(readline):
217 232 toktypes.add(t[0])
218 233 except tokenize.TokenError:
219 234 pass
220 235 return(tokenize.COMMENT in toktypes)
221 236
222 237
223 @stateless_input_transformer
238 @StatelessInputTransformer.wrap
224 239 def help_end(line):
225 240 """Translate lines with ?/?? at the end"""
226 241 m = _help_end_re.search(line)
227 242 if m is None or has_comment(line):
228 243 return line
229 244 target = m.group(1)
230 245 esc = m.group(3)
231 246 lspace = _initial_space_re.match(line).group(0)
232 247
233 248 # If we're mid-command, put it back on the next prompt for the user.
234 249 next_input = line.rstrip('?') if line.strip() != m.group(0) else None
235 250
236 251 return _make_help_call(target, esc, lspace, next_input)
237 252
238 253
239 @coroutine_input_transformer
254 @CoroutineInputTransformer.wrap
240 255 def cellmagic():
241 256 """Captures & transforms cell magics.
242 257
243 258 After a cell magic is started, this stores up any lines it gets until it is
244 259 reset (sent None).
245 260 """
246 261 tpl = 'get_ipython().run_cell_magic(%r, %r, %r)'
247 262 cellmagic_help_re = re.compile('%%\w+\?')
248 263 line = ''
249 264 while True:
250 265 line = (yield line)
251 266 if (not line) or (not line.startswith(ESC_MAGIC2)):
252 267 continue
253 268
254 269 if cellmagic_help_re.match(line):
255 270 # This case will be handled by help_end
256 271 continue
257 272
258 273 first = line
259 274 body = []
260 275 line = (yield None)
261 276 while (line is not None) and (line.strip() != ''):
262 277 body.append(line)
263 278 line = (yield None)
264 279
265 280 # Output
266 281 magic_name, _, first = first.partition(' ')
267 282 magic_name = magic_name.lstrip(ESC_MAGIC2)
268 283 line = tpl % (magic_name, first, u'\n'.join(body))
269 284
270 285
271 286 def _strip_prompts(prompt1_re, prompt2_re):
272 287 """Remove matching input prompts from a block of input."""
273 288 line = ''
274 289 while True:
275 290 line = (yield line)
276 291
277 292 if line is None:
278 293 continue
279 294
280 295 m = prompt1_re.match(line)
281 296 if m:
282 297 while m:
283 298 line = (yield line[len(m.group(0)):])
284 299 if line is None:
285 300 break
286 301 m = prompt2_re.match(line)
287 302 else:
288 303 # Prompts not in input - wait for reset
289 304 while line is not None:
290 305 line = (yield line)
291 306
292 @coroutine_input_transformer
307 @CoroutineInputTransformer.wrap
293 308 def classic_prompt():
294 309 """Strip the >>>/... prompts of the Python interactive shell."""
295 310 prompt1_re = re.compile(r'^(>>> )')
296 311 prompt2_re = re.compile(r'^(>>> |^\.\.\. )')
297 312 return _strip_prompts(prompt1_re, prompt2_re)
298 313
299 314 classic_prompt.look_in_string = True
300 315
301 @coroutine_input_transformer
316 @CoroutineInputTransformer.wrap
302 317 def ipy_prompt():
303 318 """Strip IPython's In [1]:/...: prompts."""
304 319 prompt1_re = re.compile(r'^In \[\d+\]: ')
305 320 prompt2_re = re.compile(r'^(In \[\d+\]: |^\ \ \ \.\.\.+: )')
306 321 return _strip_prompts(prompt1_re, prompt2_re)
307 322
308 323 ipy_prompt.look_in_string = True
309 324
310 325
311 @coroutine_input_transformer
326 @CoroutineInputTransformer.wrap
312 327 def leading_indent():
313 328 """Remove leading indentation.
314 329
315 330 If the first line starts with a spaces or tabs, the same whitespace will be
316 331 removed from each following line until it is reset.
317 332 """
318 333 space_re = re.compile(r'^[ \t]+')
319 334 line = ''
320 335 while True:
321 336 line = (yield line)
322 337
323 338 if line is None:
324 339 continue
325 340
326 341 m = space_re.match(line)
327 342 if m:
328 343 space = m.group(0)
329 344 while line is not None:
330 345 if line.startswith(space):
331 346 line = line[len(space):]
332 347 line = (yield line)
333 348 else:
334 349 # No leading spaces - wait for reset
335 350 while line is not None:
336 351 line = (yield line)
337 352
338 353 leading_indent.look_in_string = True
339 354
340 355
341 356 def _special_assignment(assignment_re, template):
342 357 """Transform assignment from system & magic commands.
343 358
344 359 This is stateful so that it can handle magic commands continued on several
345 360 lines.
346 361 """
347 362 line = ''
348 363 while True:
349 364 line = (yield line)
350 365 if not line or line.isspace():
351 366 continue
352 367
353 368 m = assignment_re.match(line)
354 369 if not m:
355 370 continue
356 371
357 372 parts = []
358 373 while line is not None:
359 374 parts.append(line.rstrip('\\'))
360 375 if not line.endswith('\\'):
361 376 break
362 377 line = (yield None)
363 378
364 379 # Output
365 380 whole = assignment_re.match(' '.join(parts))
366 381 line = template % (whole.group('lhs'), whole.group('cmd'))
367 382
368 @coroutine_input_transformer
383 @CoroutineInputTransformer.wrap
369 384 def assign_from_system():
370 385 """Transform assignment from system commands (e.g. files = !ls)"""
371 386 assignment_re = re.compile(r'(?P<lhs>(\s*)([\w\.]+)((\s*,\s*[\w\.]+)*))'
372 387 r'\s*=\s*!\s*(?P<cmd>.*)')
373 388 template = '%s = get_ipython().getoutput(%r)'
374 389 return _special_assignment(assignment_re, template)
375 390
376 @coroutine_input_transformer
391 @CoroutineInputTransformer.wrap
377 392 def assign_from_magic():
378 393 """Transform assignment from magic commands (e.g. a = %who_ls)"""
379 394 assignment_re = re.compile(r'(?P<lhs>(\s*)([\w\.]+)((\s*,\s*[\w\.]+)*))'
380 395 r'\s*=\s*%\s*(?P<cmd>.*)')
381 396 template = '%s = get_ipython().magic(%r)'
382 397 return _special_assignment(assignment_re, template)
General Comments 0
You need to be logged in to leave comments. Login now