##// END OF EJS Templates
Backport PR #10157: Fix detection of complete input with line continuations inside a multiline string...
Matthias Bussonnier -
Show More
@@ -1,561 +1,554 b''
1 """Input transformer classes to support IPython special syntax.
1 """Input transformer classes to support IPython special syntax.
2
2
3 This includes the machinery to recognise and transform ``%magic`` commands,
3 This includes the machinery to recognise and transform ``%magic`` commands,
4 ``!system`` commands, ``help?`` querying, prompt stripping, and so forth.
4 ``!system`` commands, ``help?`` querying, prompt stripping, and so forth.
5 """
5 """
6 import abc
6 import abc
7 import functools
7 import functools
8 import re
8 import re
9
9
10 from IPython.core.splitinput import LineInfo
10 from IPython.core.splitinput import LineInfo
11 from IPython.utils import tokenize2
11 from IPython.utils import tokenize2
12 from IPython.utils.openpy import cookie_comment_re
12 from IPython.utils.openpy import cookie_comment_re
13 from IPython.utils.py3compat import with_metaclass, PY3
13 from IPython.utils.py3compat import with_metaclass, PY3
14 from IPython.utils.tokenize2 import generate_tokens, untokenize, TokenError
14 from IPython.utils.tokenize2 import generate_tokens, untokenize, TokenError
15
15
16 if PY3:
16 if PY3:
17 from io import StringIO
17 from io import StringIO
18 else:
18 else:
19 from StringIO import StringIO
19 from StringIO import StringIO
20
20
21 #-----------------------------------------------------------------------------
21 #-----------------------------------------------------------------------------
22 # Globals
22 # Globals
23 #-----------------------------------------------------------------------------
23 #-----------------------------------------------------------------------------
24
24
25 # The escape sequences that define the syntax transformations IPython will
25 # The escape sequences that define the syntax transformations IPython will
26 # apply to user input. These can NOT be just changed here: many regular
26 # apply to user input. These can NOT be just changed here: many regular
27 # expressions and other parts of the code may use their hardcoded values, and
27 # expressions and other parts of the code may use their hardcoded values, and
28 # for all intents and purposes they constitute the 'IPython syntax', so they
28 # for all intents and purposes they constitute the 'IPython syntax', so they
29 # should be considered fixed.
29 # should be considered fixed.
30
30
31 ESC_SHELL = '!' # Send line to underlying system shell
31 ESC_SHELL = '!' # Send line to underlying system shell
32 ESC_SH_CAP = '!!' # Send line to system shell and capture output
32 ESC_SH_CAP = '!!' # Send line to system shell and capture output
33 ESC_HELP = '?' # Find information about object
33 ESC_HELP = '?' # Find information about object
34 ESC_HELP2 = '??' # Find extra-detailed information about object
34 ESC_HELP2 = '??' # Find extra-detailed information about object
35 ESC_MAGIC = '%' # Call magic function
35 ESC_MAGIC = '%' # Call magic function
36 ESC_MAGIC2 = '%%' # Call cell-magic function
36 ESC_MAGIC2 = '%%' # Call cell-magic function
37 ESC_QUOTE = ',' # Split args on whitespace, quote each as string and call
37 ESC_QUOTE = ',' # Split args on whitespace, quote each as string and call
38 ESC_QUOTE2 = ';' # Quote all args as a single string, call
38 ESC_QUOTE2 = ';' # Quote all args as a single string, call
39 ESC_PAREN = '/' # Call first argument with rest of line as arguments
39 ESC_PAREN = '/' # Call first argument with rest of line as arguments
40
40
41 ESC_SEQUENCES = [ESC_SHELL, ESC_SH_CAP, ESC_HELP ,\
41 ESC_SEQUENCES = [ESC_SHELL, ESC_SH_CAP, ESC_HELP ,\
42 ESC_HELP2, ESC_MAGIC, ESC_MAGIC2,\
42 ESC_HELP2, ESC_MAGIC, ESC_MAGIC2,\
43 ESC_QUOTE, ESC_QUOTE2, ESC_PAREN ]
43 ESC_QUOTE, ESC_QUOTE2, ESC_PAREN ]
44
44
45
45
46 class InputTransformer(with_metaclass(abc.ABCMeta, object)):
46 class InputTransformer(with_metaclass(abc.ABCMeta, object)):
47 """Abstract base class for line-based input transformers."""
47 """Abstract base class for line-based input transformers."""
48
48
49 @abc.abstractmethod
49 @abc.abstractmethod
50 def push(self, line):
50 def push(self, line):
51 """Send a line of input to the transformer, returning the transformed
51 """Send a line of input to the transformer, returning the transformed
52 input or None if the transformer is waiting for more input.
52 input or None if the transformer is waiting for more input.
53
53
54 Must be overridden by subclasses.
54 Must be overridden by subclasses.
55
55
56 Implementations may raise ``SyntaxError`` if the input is invalid. No
56 Implementations may raise ``SyntaxError`` if the input is invalid. No
57 other exceptions may be raised.
57 other exceptions may be raised.
58 """
58 """
59 pass
59 pass
60
60
61 @abc.abstractmethod
61 @abc.abstractmethod
62 def reset(self):
62 def reset(self):
63 """Return, transformed any lines that the transformer has accumulated,
63 """Return, transformed any lines that the transformer has accumulated,
64 and reset its internal state.
64 and reset its internal state.
65
65
66 Must be overridden by subclasses.
66 Must be overridden by subclasses.
67 """
67 """
68 pass
68 pass
69
69
70 @classmethod
70 @classmethod
71 def wrap(cls, func):
71 def wrap(cls, func):
72 """Can be used by subclasses as a decorator, to return a factory that
72 """Can be used by subclasses as a decorator, to return a factory that
73 will allow instantiation with the decorated object.
73 will allow instantiation with the decorated object.
74 """
74 """
75 @functools.wraps(func)
75 @functools.wraps(func)
76 def transformer_factory(**kwargs):
76 def transformer_factory(**kwargs):
77 return cls(func, **kwargs)
77 return cls(func, **kwargs)
78
78
79 return transformer_factory
79 return transformer_factory
80
80
81 class StatelessInputTransformer(InputTransformer):
81 class StatelessInputTransformer(InputTransformer):
82 """Wrapper for a stateless input transformer implemented as a function."""
82 """Wrapper for a stateless input transformer implemented as a function."""
83 def __init__(self, func):
83 def __init__(self, func):
84 self.func = func
84 self.func = func
85
85
86 def __repr__(self):
86 def __repr__(self):
87 return "StatelessInputTransformer(func={0!r})".format(self.func)
87 return "StatelessInputTransformer(func={0!r})".format(self.func)
88
88
89 def push(self, line):
89 def push(self, line):
90 """Send a line of input to the transformer, returning the
90 """Send a line of input to the transformer, returning the
91 transformed input."""
91 transformed input."""
92 return self.func(line)
92 return self.func(line)
93
93
94 def reset(self):
94 def reset(self):
95 """No-op - exists for compatibility."""
95 """No-op - exists for compatibility."""
96 pass
96 pass
97
97
98 class CoroutineInputTransformer(InputTransformer):
98 class CoroutineInputTransformer(InputTransformer):
99 """Wrapper for an input transformer implemented as a coroutine."""
99 """Wrapper for an input transformer implemented as a coroutine."""
100 def __init__(self, coro, **kwargs):
100 def __init__(self, coro, **kwargs):
101 # Prime it
101 # Prime it
102 self.coro = coro(**kwargs)
102 self.coro = coro(**kwargs)
103 next(self.coro)
103 next(self.coro)
104
104
105 def __repr__(self):
105 def __repr__(self):
106 return "CoroutineInputTransformer(coro={0!r})".format(self.coro)
106 return "CoroutineInputTransformer(coro={0!r})".format(self.coro)
107
107
108 def push(self, line):
108 def push(self, line):
109 """Send a line of input to the transformer, returning the
109 """Send a line of input to the transformer, returning the
110 transformed input or None if the transformer is waiting for more
110 transformed input or None if the transformer is waiting for more
111 input.
111 input.
112 """
112 """
113 return self.coro.send(line)
113 return self.coro.send(line)
114
114
115 def reset(self):
115 def reset(self):
116 """Return, transformed any lines that the transformer has
116 """Return, transformed any lines that the transformer has
117 accumulated, and reset its internal state.
117 accumulated, and reset its internal state.
118 """
118 """
119 return self.coro.send(None)
119 return self.coro.send(None)
120
120
121 class TokenInputTransformer(InputTransformer):
121 class TokenInputTransformer(InputTransformer):
122 """Wrapper for a token-based input transformer.
122 """Wrapper for a token-based input transformer.
123
123
124 func should accept a list of tokens (5-tuples, see tokenize docs), and
124 func should accept a list of tokens (5-tuples, see tokenize docs), and
125 return an iterable which can be passed to tokenize.untokenize().
125 return an iterable which can be passed to tokenize.untokenize().
126 """
126 """
127 def __init__(self, func):
127 def __init__(self, func):
128 self.func = func
128 self.func = func
129 self.current_line = ""
129 self.buf = []
130 self.line_used = False
131 self.reset_tokenizer()
130 self.reset_tokenizer()
132
131
133 def reset_tokenizer(self):
132 def reset_tokenizer(self):
134 self.tokenizer = generate_tokens(self.get_line)
133 it = iter(self.buf)
135
134 self.tokenizer = generate_tokens(it.__next__)
136 def get_line(self):
135
137 if self.line_used:
138 raise TokenError
139 self.line_used = True
140 return self.current_line
141
142 def push(self, line):
136 def push(self, line):
143 self.current_line += line + "\n"
137 self.buf.append(line + '\n')
144 if self.current_line.isspace():
138 if all(l.isspace() for l in self.buf):
145 return self.reset()
139 return self.reset()
146
140
147 self.line_used = False
148 tokens = []
141 tokens = []
149 stop_at_NL = False
142 stop_at_NL = False
150 try:
143 try:
151 for intok in self.tokenizer:
144 for intok in self.tokenizer:
152 tokens.append(intok)
145 tokens.append(intok)
153 t = intok[0]
146 t = intok[0]
154 if t == tokenize2.NEWLINE or (stop_at_NL and t == tokenize2.NL):
147 if t == tokenize2.NEWLINE or (stop_at_NL and t == tokenize2.NL):
155 # Stop before we try to pull a line we don't have yet
148 # Stop before we try to pull a line we don't have yet
156 break
149 break
157 elif t == tokenize2.ERRORTOKEN:
150 elif t == tokenize2.ERRORTOKEN:
158 stop_at_NL = True
151 stop_at_NL = True
159 except TokenError:
152 except TokenError:
160 # Multi-line statement - stop and try again with the next line
153 # Multi-line statement - stop and try again with the next line
161 self.reset_tokenizer()
154 self.reset_tokenizer()
162 return None
155 return None
163
156
164 return self.output(tokens)
157 return self.output(tokens)
165
158
166 def output(self, tokens):
159 def output(self, tokens):
167 self.current_line = ""
160 self.buf.clear()
168 self.reset_tokenizer()
161 self.reset_tokenizer()
169 return untokenize(self.func(tokens)).rstrip('\n')
162 return untokenize(self.func(tokens)).rstrip('\n')
170
163
171 def reset(self):
164 def reset(self):
172 l = self.current_line
165 l = ''.join(self.buf)
173 self.current_line = ""
166 self.buf.clear()
174 self.reset_tokenizer()
167 self.reset_tokenizer()
175 if l:
168 if l:
176 return l.rstrip('\n')
169 return l.rstrip('\n')
177
170
178 class assemble_python_lines(TokenInputTransformer):
171 class assemble_python_lines(TokenInputTransformer):
179 def __init__(self):
172 def __init__(self):
180 super(assemble_python_lines, self).__init__(None)
173 super(assemble_python_lines, self).__init__(None)
181
174
182 def output(self, tokens):
175 def output(self, tokens):
183 return self.reset()
176 return self.reset()
184
177
185 @CoroutineInputTransformer.wrap
178 @CoroutineInputTransformer.wrap
186 def assemble_logical_lines():
179 def assemble_logical_lines():
187 """Join lines following explicit line continuations (\)"""
180 """Join lines following explicit line continuations (\)"""
188 line = ''
181 line = ''
189 while True:
182 while True:
190 line = (yield line)
183 line = (yield line)
191 if not line or line.isspace():
184 if not line or line.isspace():
192 continue
185 continue
193
186
194 parts = []
187 parts = []
195 while line is not None:
188 while line is not None:
196 if line.endswith('\\') and (not has_comment(line)):
189 if line.endswith('\\') and (not has_comment(line)):
197 parts.append(line[:-1])
190 parts.append(line[:-1])
198 line = (yield None) # Get another line
191 line = (yield None) # Get another line
199 else:
192 else:
200 parts.append(line)
193 parts.append(line)
201 break
194 break
202
195
203 # Output
196 # Output
204 line = ''.join(parts)
197 line = ''.join(parts)
205
198
206 # Utilities
199 # Utilities
207 def _make_help_call(target, esc, lspace, next_input=None):
200 def _make_help_call(target, esc, lspace, next_input=None):
208 """Prepares a pinfo(2)/psearch call from a target name and the escape
201 """Prepares a pinfo(2)/psearch call from a target name and the escape
209 (i.e. ? or ??)"""
202 (i.e. ? or ??)"""
210 method = 'pinfo2' if esc == '??' \
203 method = 'pinfo2' if esc == '??' \
211 else 'psearch' if '*' in target \
204 else 'psearch' if '*' in target \
212 else 'pinfo'
205 else 'pinfo'
213 arg = " ".join([method, target])
206 arg = " ".join([method, target])
214 if next_input is None:
207 if next_input is None:
215 return '%sget_ipython().magic(%r)' % (lspace, arg)
208 return '%sget_ipython().magic(%r)' % (lspace, arg)
216 else:
209 else:
217 return '%sget_ipython().set_next_input(%r);get_ipython().magic(%r)' % \
210 return '%sget_ipython().set_next_input(%r);get_ipython().magic(%r)' % \
218 (lspace, next_input, arg)
211 (lspace, next_input, arg)
219
212
220 # These define the transformations for the different escape characters.
213 # These define the transformations for the different escape characters.
221 def _tr_system(line_info):
214 def _tr_system(line_info):
222 "Translate lines escaped with: !"
215 "Translate lines escaped with: !"
223 cmd = line_info.line.lstrip().lstrip(ESC_SHELL)
216 cmd = line_info.line.lstrip().lstrip(ESC_SHELL)
224 return '%sget_ipython().system(%r)' % (line_info.pre, cmd)
217 return '%sget_ipython().system(%r)' % (line_info.pre, cmd)
225
218
226 def _tr_system2(line_info):
219 def _tr_system2(line_info):
227 "Translate lines escaped with: !!"
220 "Translate lines escaped with: !!"
228 cmd = line_info.line.lstrip()[2:]
221 cmd = line_info.line.lstrip()[2:]
229 return '%sget_ipython().getoutput(%r)' % (line_info.pre, cmd)
222 return '%sget_ipython().getoutput(%r)' % (line_info.pre, cmd)
230
223
231 def _tr_help(line_info):
224 def _tr_help(line_info):
232 "Translate lines escaped with: ?/??"
225 "Translate lines escaped with: ?/??"
233 # A naked help line should just fire the intro help screen
226 # A naked help line should just fire the intro help screen
234 if not line_info.line[1:]:
227 if not line_info.line[1:]:
235 return 'get_ipython().show_usage()'
228 return 'get_ipython().show_usage()'
236
229
237 return _make_help_call(line_info.ifun, line_info.esc, line_info.pre)
230 return _make_help_call(line_info.ifun, line_info.esc, line_info.pre)
238
231
239 def _tr_magic(line_info):
232 def _tr_magic(line_info):
240 "Translate lines escaped with: %"
233 "Translate lines escaped with: %"
241 tpl = '%sget_ipython().magic(%r)'
234 tpl = '%sget_ipython().magic(%r)'
242 if line_info.line.startswith(ESC_MAGIC2):
235 if line_info.line.startswith(ESC_MAGIC2):
243 return line_info.line
236 return line_info.line
244 cmd = ' '.join([line_info.ifun, line_info.the_rest]).strip()
237 cmd = ' '.join([line_info.ifun, line_info.the_rest]).strip()
245 return tpl % (line_info.pre, cmd)
238 return tpl % (line_info.pre, cmd)
246
239
247 def _tr_quote(line_info):
240 def _tr_quote(line_info):
248 "Translate lines escaped with: ,"
241 "Translate lines escaped with: ,"
249 return '%s%s("%s")' % (line_info.pre, line_info.ifun,
242 return '%s%s("%s")' % (line_info.pre, line_info.ifun,
250 '", "'.join(line_info.the_rest.split()) )
243 '", "'.join(line_info.the_rest.split()) )
251
244
252 def _tr_quote2(line_info):
245 def _tr_quote2(line_info):
253 "Translate lines escaped with: ;"
246 "Translate lines escaped with: ;"
254 return '%s%s("%s")' % (line_info.pre, line_info.ifun,
247 return '%s%s("%s")' % (line_info.pre, line_info.ifun,
255 line_info.the_rest)
248 line_info.the_rest)
256
249
257 def _tr_paren(line_info):
250 def _tr_paren(line_info):
258 "Translate lines escaped with: /"
251 "Translate lines escaped with: /"
259 return '%s%s(%s)' % (line_info.pre, line_info.ifun,
252 return '%s%s(%s)' % (line_info.pre, line_info.ifun,
260 ", ".join(line_info.the_rest.split()))
253 ", ".join(line_info.the_rest.split()))
261
254
262 tr = { ESC_SHELL : _tr_system,
255 tr = { ESC_SHELL : _tr_system,
263 ESC_SH_CAP : _tr_system2,
256 ESC_SH_CAP : _tr_system2,
264 ESC_HELP : _tr_help,
257 ESC_HELP : _tr_help,
265 ESC_HELP2 : _tr_help,
258 ESC_HELP2 : _tr_help,
266 ESC_MAGIC : _tr_magic,
259 ESC_MAGIC : _tr_magic,
267 ESC_QUOTE : _tr_quote,
260 ESC_QUOTE : _tr_quote,
268 ESC_QUOTE2 : _tr_quote2,
261 ESC_QUOTE2 : _tr_quote2,
269 ESC_PAREN : _tr_paren }
262 ESC_PAREN : _tr_paren }
270
263
271 @StatelessInputTransformer.wrap
264 @StatelessInputTransformer.wrap
272 def escaped_commands(line):
265 def escaped_commands(line):
273 """Transform escaped commands - %magic, !system, ?help + various autocalls.
266 """Transform escaped commands - %magic, !system, ?help + various autocalls.
274 """
267 """
275 if not line or line.isspace():
268 if not line or line.isspace():
276 return line
269 return line
277 lineinf = LineInfo(line)
270 lineinf = LineInfo(line)
278 if lineinf.esc not in tr:
271 if lineinf.esc not in tr:
279 return line
272 return line
280
273
281 return tr[lineinf.esc](lineinf)
274 return tr[lineinf.esc](lineinf)
282
275
283 _initial_space_re = re.compile(r'\s*')
276 _initial_space_re = re.compile(r'\s*')
284
277
285 _help_end_re = re.compile(r"""(%{0,2}
278 _help_end_re = re.compile(r"""(%{0,2}
286 [a-zA-Z_*][\w*]* # Variable name
279 [a-zA-Z_*][\w*]* # Variable name
287 (\.[a-zA-Z_*][\w*]*)* # .etc.etc
280 (\.[a-zA-Z_*][\w*]*)* # .etc.etc
288 )
281 )
289 (\?\??)$ # ? or ??
282 (\?\??)$ # ? or ??
290 """,
283 """,
291 re.VERBOSE)
284 re.VERBOSE)
292
285
293 # Extra pseudotokens for multiline strings and data structures
286 # Extra pseudotokens for multiline strings and data structures
294 _MULTILINE_STRING = object()
287 _MULTILINE_STRING = object()
295 _MULTILINE_STRUCTURE = object()
288 _MULTILINE_STRUCTURE = object()
296
289
297 def _line_tokens(line):
290 def _line_tokens(line):
298 """Helper for has_comment and ends_in_comment_or_string."""
291 """Helper for has_comment and ends_in_comment_or_string."""
299 readline = StringIO(line).readline
292 readline = StringIO(line).readline
300 toktypes = set()
293 toktypes = set()
301 try:
294 try:
302 for t in generate_tokens(readline):
295 for t in generate_tokens(readline):
303 toktypes.add(t[0])
296 toktypes.add(t[0])
304 except TokenError as e:
297 except TokenError as e:
305 # There are only two cases where a TokenError is raised.
298 # There are only two cases where a TokenError is raised.
306 if 'multi-line string' in e.args[0]:
299 if 'multi-line string' in e.args[0]:
307 toktypes.add(_MULTILINE_STRING)
300 toktypes.add(_MULTILINE_STRING)
308 else:
301 else:
309 toktypes.add(_MULTILINE_STRUCTURE)
302 toktypes.add(_MULTILINE_STRUCTURE)
310 return toktypes
303 return toktypes
311
304
312 def has_comment(src):
305 def has_comment(src):
313 """Indicate whether an input line has (i.e. ends in, or is) a comment.
306 """Indicate whether an input line has (i.e. ends in, or is) a comment.
314
307
315 This uses tokenize, so it can distinguish comments from # inside strings.
308 This uses tokenize, so it can distinguish comments from # inside strings.
316
309
317 Parameters
310 Parameters
318 ----------
311 ----------
319 src : string
312 src : string
320 A single line input string.
313 A single line input string.
321
314
322 Returns
315 Returns
323 -------
316 -------
324 comment : bool
317 comment : bool
325 True if source has a comment.
318 True if source has a comment.
326 """
319 """
327 return (tokenize2.COMMENT in _line_tokens(src))
320 return (tokenize2.COMMENT in _line_tokens(src))
328
321
329 def ends_in_comment_or_string(src):
322 def ends_in_comment_or_string(src):
330 """Indicates whether or not an input line ends in a comment or within
323 """Indicates whether or not an input line ends in a comment or within
331 a multiline string.
324 a multiline string.
332
325
333 Parameters
326 Parameters
334 ----------
327 ----------
335 src : string
328 src : string
336 A single line input string.
329 A single line input string.
337
330
338 Returns
331 Returns
339 -------
332 -------
340 comment : bool
333 comment : bool
341 True if source ends in a comment or multiline string.
334 True if source ends in a comment or multiline string.
342 """
335 """
343 toktypes = _line_tokens(src)
336 toktypes = _line_tokens(src)
344 return (tokenize2.COMMENT in toktypes) or (_MULTILINE_STRING in toktypes)
337 return (tokenize2.COMMENT in toktypes) or (_MULTILINE_STRING in toktypes)
345
338
346
339
347 @StatelessInputTransformer.wrap
340 @StatelessInputTransformer.wrap
348 def help_end(line):
341 def help_end(line):
349 """Translate lines with ?/?? at the end"""
342 """Translate lines with ?/?? at the end"""
350 m = _help_end_re.search(line)
343 m = _help_end_re.search(line)
351 if m is None or ends_in_comment_or_string(line):
344 if m is None or ends_in_comment_or_string(line):
352 return line
345 return line
353 target = m.group(1)
346 target = m.group(1)
354 esc = m.group(3)
347 esc = m.group(3)
355 lspace = _initial_space_re.match(line).group(0)
348 lspace = _initial_space_re.match(line).group(0)
356
349
357 # If we're mid-command, put it back on the next prompt for the user.
350 # If we're mid-command, put it back on the next prompt for the user.
358 next_input = line.rstrip('?') if line.strip() != m.group(0) else None
351 next_input = line.rstrip('?') if line.strip() != m.group(0) else None
359
352
360 return _make_help_call(target, esc, lspace, next_input)
353 return _make_help_call(target, esc, lspace, next_input)
361
354
362
355
363 @CoroutineInputTransformer.wrap
356 @CoroutineInputTransformer.wrap
364 def cellmagic(end_on_blank_line=False):
357 def cellmagic(end_on_blank_line=False):
365 """Captures & transforms cell magics.
358 """Captures & transforms cell magics.
366
359
367 After a cell magic is started, this stores up any lines it gets until it is
360 After a cell magic is started, this stores up any lines it gets until it is
368 reset (sent None).
361 reset (sent None).
369 """
362 """
370 tpl = 'get_ipython().run_cell_magic(%r, %r, %r)'
363 tpl = 'get_ipython().run_cell_magic(%r, %r, %r)'
371 cellmagic_help_re = re.compile('%%\w+\?')
364 cellmagic_help_re = re.compile('%%\w+\?')
372 line = ''
365 line = ''
373 while True:
366 while True:
374 line = (yield line)
367 line = (yield line)
375 # consume leading empty lines
368 # consume leading empty lines
376 while not line:
369 while not line:
377 line = (yield line)
370 line = (yield line)
378
371
379 if not line.startswith(ESC_MAGIC2):
372 if not line.startswith(ESC_MAGIC2):
380 # This isn't a cell magic, idle waiting for reset then start over
373 # This isn't a cell magic, idle waiting for reset then start over
381 while line is not None:
374 while line is not None:
382 line = (yield line)
375 line = (yield line)
383 continue
376 continue
384
377
385 if cellmagic_help_re.match(line):
378 if cellmagic_help_re.match(line):
386 # This case will be handled by help_end
379 # This case will be handled by help_end
387 continue
380 continue
388
381
389 first = line
382 first = line
390 body = []
383 body = []
391 line = (yield None)
384 line = (yield None)
392 while (line is not None) and \
385 while (line is not None) and \
393 ((line.strip() != '') or not end_on_blank_line):
386 ((line.strip() != '') or not end_on_blank_line):
394 body.append(line)
387 body.append(line)
395 line = (yield None)
388 line = (yield None)
396
389
397 # Output
390 # Output
398 magic_name, _, first = first.partition(' ')
391 magic_name, _, first = first.partition(' ')
399 magic_name = magic_name.lstrip(ESC_MAGIC2)
392 magic_name = magic_name.lstrip(ESC_MAGIC2)
400 line = tpl % (magic_name, first, u'\n'.join(body))
393 line = tpl % (magic_name, first, u'\n'.join(body))
401
394
402
395
403 def _strip_prompts(prompt_re, initial_re=None, turnoff_re=None):
396 def _strip_prompts(prompt_re, initial_re=None, turnoff_re=None):
404 """Remove matching input prompts from a block of input.
397 """Remove matching input prompts from a block of input.
405
398
406 Parameters
399 Parameters
407 ----------
400 ----------
408 prompt_re : regular expression
401 prompt_re : regular expression
409 A regular expression matching any input prompt (including continuation)
402 A regular expression matching any input prompt (including continuation)
410 initial_re : regular expression, optional
403 initial_re : regular expression, optional
411 A regular expression matching only the initial prompt, but not continuation.
404 A regular expression matching only the initial prompt, but not continuation.
412 If no initial expression is given, prompt_re will be used everywhere.
405 If no initial expression is given, prompt_re will be used everywhere.
413 Used mainly for plain Python prompts, where the continuation prompt
406 Used mainly for plain Python prompts, where the continuation prompt
414 ``...`` is a valid Python expression in Python 3, so shouldn't be stripped.
407 ``...`` is a valid Python expression in Python 3, so shouldn't be stripped.
415
408
416 If initial_re and prompt_re differ,
409 If initial_re and prompt_re differ,
417 only initial_re will be tested against the first line.
410 only initial_re will be tested against the first line.
418 If any prompt is found on the first two lines,
411 If any prompt is found on the first two lines,
419 prompts will be stripped from the rest of the block.
412 prompts will be stripped from the rest of the block.
420 """
413 """
421 if initial_re is None:
414 if initial_re is None:
422 initial_re = prompt_re
415 initial_re = prompt_re
423 line = ''
416 line = ''
424 while True:
417 while True:
425 line = (yield line)
418 line = (yield line)
426
419
427 # First line of cell
420 # First line of cell
428 if line is None:
421 if line is None:
429 continue
422 continue
430 out, n1 = initial_re.subn('', line, count=1)
423 out, n1 = initial_re.subn('', line, count=1)
431 if turnoff_re and not n1:
424 if turnoff_re and not n1:
432 if turnoff_re.match(line):
425 if turnoff_re.match(line):
433 # We're in e.g. a cell magic; disable this transformer for
426 # We're in e.g. a cell magic; disable this transformer for
434 # the rest of the cell.
427 # the rest of the cell.
435 while line is not None:
428 while line is not None:
436 line = (yield line)
429 line = (yield line)
437 continue
430 continue
438
431
439 line = (yield out)
432 line = (yield out)
440
433
441 if line is None:
434 if line is None:
442 continue
435 continue
443 # check for any prompt on the second line of the cell,
436 # check for any prompt on the second line of the cell,
444 # because people often copy from just after the first prompt,
437 # because people often copy from just after the first prompt,
445 # so we might not see it in the first line.
438 # so we might not see it in the first line.
446 out, n2 = prompt_re.subn('', line, count=1)
439 out, n2 = prompt_re.subn('', line, count=1)
447 line = (yield out)
440 line = (yield out)
448
441
449 if n1 or n2:
442 if n1 or n2:
450 # Found a prompt in the first two lines - check for it in
443 # Found a prompt in the first two lines - check for it in
451 # the rest of the cell as well.
444 # the rest of the cell as well.
452 while line is not None:
445 while line is not None:
453 line = (yield prompt_re.sub('', line, count=1))
446 line = (yield prompt_re.sub('', line, count=1))
454
447
455 else:
448 else:
456 # Prompts not in input - wait for reset
449 # Prompts not in input - wait for reset
457 while line is not None:
450 while line is not None:
458 line = (yield line)
451 line = (yield line)
459
452
460 @CoroutineInputTransformer.wrap
453 @CoroutineInputTransformer.wrap
461 def classic_prompt():
454 def classic_prompt():
462 """Strip the >>>/... prompts of the Python interactive shell."""
455 """Strip the >>>/... prompts of the Python interactive shell."""
463 # FIXME: non-capturing version (?:...) usable?
456 # FIXME: non-capturing version (?:...) usable?
464 prompt_re = re.compile(r'^(>>>|\.\.\.)( |$)')
457 prompt_re = re.compile(r'^(>>>|\.\.\.)( |$)')
465 initial_re = re.compile(r'^>>>( |$)')
458 initial_re = re.compile(r'^>>>( |$)')
466 # Any %magic/!system is IPython syntax, so we needn't look for >>> prompts
459 # Any %magic/!system is IPython syntax, so we needn't look for >>> prompts
467 turnoff_re = re.compile(r'^[%!]')
460 turnoff_re = re.compile(r'^[%!]')
468 return _strip_prompts(prompt_re, initial_re, turnoff_re)
461 return _strip_prompts(prompt_re, initial_re, turnoff_re)
469
462
470 @CoroutineInputTransformer.wrap
463 @CoroutineInputTransformer.wrap
471 def ipy_prompt():
464 def ipy_prompt():
472 """Strip IPython's In [1]:/...: prompts."""
465 """Strip IPython's In [1]:/...: prompts."""
473 # FIXME: non-capturing version (?:...) usable?
466 # FIXME: non-capturing version (?:...) usable?
474 prompt_re = re.compile(r'^(In \[\d+\]: |\s*\.{3,}: ?)')
467 prompt_re = re.compile(r'^(In \[\d+\]: |\s*\.{3,}: ?)')
475 # Disable prompt stripping inside cell magics
468 # Disable prompt stripping inside cell magics
476 turnoff_re = re.compile(r'^%%')
469 turnoff_re = re.compile(r'^%%')
477 return _strip_prompts(prompt_re, turnoff_re=turnoff_re)
470 return _strip_prompts(prompt_re, turnoff_re=turnoff_re)
478
471
479
472
480 @CoroutineInputTransformer.wrap
473 @CoroutineInputTransformer.wrap
481 def leading_indent():
474 def leading_indent():
482 """Remove leading indentation.
475 """Remove leading indentation.
483
476
484 If the first line starts with a spaces or tabs, the same whitespace will be
477 If the first line starts with a spaces or tabs, the same whitespace will be
485 removed from each following line until it is reset.
478 removed from each following line until it is reset.
486 """
479 """
487 space_re = re.compile(r'^[ \t]+')
480 space_re = re.compile(r'^[ \t]+')
488 line = ''
481 line = ''
489 while True:
482 while True:
490 line = (yield line)
483 line = (yield line)
491
484
492 if line is None:
485 if line is None:
493 continue
486 continue
494
487
495 m = space_re.match(line)
488 m = space_re.match(line)
496 if m:
489 if m:
497 space = m.group(0)
490 space = m.group(0)
498 while line is not None:
491 while line is not None:
499 if line.startswith(space):
492 if line.startswith(space):
500 line = line[len(space):]
493 line = line[len(space):]
501 line = (yield line)
494 line = (yield line)
502 else:
495 else:
503 # No leading spaces - wait for reset
496 # No leading spaces - wait for reset
504 while line is not None:
497 while line is not None:
505 line = (yield line)
498 line = (yield line)
506
499
507
500
508 @CoroutineInputTransformer.wrap
501 @CoroutineInputTransformer.wrap
509 def strip_encoding_cookie():
502 def strip_encoding_cookie():
510 """Remove encoding comment if found in first two lines
503 """Remove encoding comment if found in first two lines
511
504
512 If the first or second line has the `# coding: utf-8` comment,
505 If the first or second line has the `# coding: utf-8` comment,
513 it will be removed.
506 it will be removed.
514 """
507 """
515 line = ''
508 line = ''
516 while True:
509 while True:
517 line = (yield line)
510 line = (yield line)
518 # check comment on first two lines
511 # check comment on first two lines
519 for i in range(2):
512 for i in range(2):
520 if line is None:
513 if line is None:
521 break
514 break
522 if cookie_comment_re.match(line):
515 if cookie_comment_re.match(line):
523 line = (yield "")
516 line = (yield "")
524 else:
517 else:
525 line = (yield line)
518 line = (yield line)
526
519
527 # no-op on the rest of the cell
520 # no-op on the rest of the cell
528 while line is not None:
521 while line is not None:
529 line = (yield line)
522 line = (yield line)
530
523
531 _assign_pat = \
524 _assign_pat = \
532 r'''(?P<lhs>(\s*)
525 r'''(?P<lhs>(\s*)
533 ([\w\.]+) # Initial identifier
526 ([\w\.]+) # Initial identifier
534 (\s*,\s*
527 (\s*,\s*
535 \*?[\w\.]+)* # Further identifiers for unpacking
528 \*?[\w\.]+)* # Further identifiers for unpacking
536 \s*?,? # Trailing comma
529 \s*?,? # Trailing comma
537 )
530 )
538 \s*=\s*
531 \s*=\s*
539 '''
532 '''
540
533
541 assign_system_re = re.compile(r'{}!\s*(?P<cmd>.*)'.format(_assign_pat), re.VERBOSE)
534 assign_system_re = re.compile(r'{}!\s*(?P<cmd>.*)'.format(_assign_pat), re.VERBOSE)
542 assign_system_template = '%s = get_ipython().getoutput(%r)'
535 assign_system_template = '%s = get_ipython().getoutput(%r)'
543 @StatelessInputTransformer.wrap
536 @StatelessInputTransformer.wrap
544 def assign_from_system(line):
537 def assign_from_system(line):
545 """Transform assignment from system commands (e.g. files = !ls)"""
538 """Transform assignment from system commands (e.g. files = !ls)"""
546 m = assign_system_re.match(line)
539 m = assign_system_re.match(line)
547 if m is None:
540 if m is None:
548 return line
541 return line
549
542
550 return assign_system_template % m.group('lhs', 'cmd')
543 return assign_system_template % m.group('lhs', 'cmd')
551
544
552 assign_magic_re = re.compile(r'{}%\s*(?P<cmd>.*)'.format(_assign_pat), re.VERBOSE)
545 assign_magic_re = re.compile(r'{}%\s*(?P<cmd>.*)'.format(_assign_pat), re.VERBOSE)
553 assign_magic_template = '%s = get_ipython().magic(%r)'
546 assign_magic_template = '%s = get_ipython().magic(%r)'
554 @StatelessInputTransformer.wrap
547 @StatelessInputTransformer.wrap
555 def assign_from_magic(line):
548 def assign_from_magic(line):
556 """Transform assignment from magic commands (e.g. a = %who_ls)"""
549 """Transform assignment from magic commands (e.g. a = %who_ls)"""
557 m = assign_magic_re.match(line)
550 m = assign_magic_re.match(line)
558 if m is None:
551 if m is None:
559 return line
552 return line
560
553
561 return assign_magic_template % m.group('lhs', 'cmd')
554 return assign_magic_template % m.group('lhs', 'cmd')
@@ -1,515 +1,520 b''
1 import tokenize
1 import tokenize
2 import nose.tools as nt
2 import nose.tools as nt
3
3
4 from IPython.testing import tools as tt
4 from IPython.testing import tools as tt
5 from IPython.utils import py3compat
5 from IPython.utils import py3compat
6 u_fmt = py3compat.u_format
6 u_fmt = py3compat.u_format
7
7
8 from IPython.core import inputtransformer as ipt
8 from IPython.core import inputtransformer as ipt
9
9
10 def transform_and_reset(transformer):
10 def transform_and_reset(transformer):
11 transformer = transformer()
11 transformer = transformer()
12 def transform(inp):
12 def transform(inp):
13 try:
13 try:
14 return transformer.push(inp)
14 return transformer.push(inp)
15 finally:
15 finally:
16 transformer.reset()
16 transformer.reset()
17
17
18 return transform
18 return transform
19
19
20 # Transformer tests
20 # Transformer tests
21 def transform_checker(tests, transformer, **kwargs):
21 def transform_checker(tests, transformer, **kwargs):
22 """Utility to loop over test inputs"""
22 """Utility to loop over test inputs"""
23 transformer = transformer(**kwargs)
23 transformer = transformer(**kwargs)
24 try:
24 try:
25 for inp, tr in tests:
25 for inp, tr in tests:
26 if inp is None:
26 if inp is None:
27 out = transformer.reset()
27 out = transformer.reset()
28 else:
28 else:
29 out = transformer.push(inp)
29 out = transformer.push(inp)
30 nt.assert_equal(out, tr)
30 nt.assert_equal(out, tr)
31 finally:
31 finally:
32 transformer.reset()
32 transformer.reset()
33
33
34 # Data for all the syntax tests in the form of lists of pairs of
34 # Data for all the syntax tests in the form of lists of pairs of
35 # raw/transformed input. We store it here as a global dict so that we can use
35 # raw/transformed input. We store it here as a global dict so that we can use
36 # it both within single-function tests and also to validate the behavior of the
36 # it both within single-function tests and also to validate the behavior of the
37 # larger objects
37 # larger objects
38
38
39 syntax = \
39 syntax = \
40 dict(assign_system =
40 dict(assign_system =
41 [(i,py3compat.u_format(o)) for i,o in \
41 [(i,py3compat.u_format(o)) for i,o in \
42 [(u'a =! ls', "a = get_ipython().getoutput({u}'ls')"),
42 [(u'a =! ls', "a = get_ipython().getoutput({u}'ls')"),
43 (u'b = !ls', "b = get_ipython().getoutput({u}'ls')"),
43 (u'b = !ls', "b = get_ipython().getoutput({u}'ls')"),
44 (u'c= !ls', "c = get_ipython().getoutput({u}'ls')"),
44 (u'c= !ls', "c = get_ipython().getoutput({u}'ls')"),
45 (u'd == !ls', u'd == !ls'), # Invalid syntax, but we leave == alone.
45 (u'd == !ls', u'd == !ls'), # Invalid syntax, but we leave == alone.
46 ('x=1', 'x=1'), # normal input is unmodified
46 ('x=1', 'x=1'), # normal input is unmodified
47 (' ',' '), # blank lines are kept intact
47 (' ',' '), # blank lines are kept intact
48 # Tuple unpacking
48 # Tuple unpacking
49 (u"a, b = !echo 'a\\nb'", u"a, b = get_ipython().getoutput({u}\"echo 'a\\\\nb'\")"),
49 (u"a, b = !echo 'a\\nb'", u"a, b = get_ipython().getoutput({u}\"echo 'a\\\\nb'\")"),
50 (u"a,= !echo 'a'", u"a, = get_ipython().getoutput({u}\"echo 'a'\")"),
50 (u"a,= !echo 'a'", u"a, = get_ipython().getoutput({u}\"echo 'a'\")"),
51 (u"a, *bc = !echo 'a\\nb\\nc'", u"a, *bc = get_ipython().getoutput({u}\"echo 'a\\\\nb\\\\nc'\")"),
51 (u"a, *bc = !echo 'a\\nb\\nc'", u"a, *bc = get_ipython().getoutput({u}\"echo 'a\\\\nb\\\\nc'\")"),
52 # Tuple unpacking with regular Python expressions, not our syntax.
52 # Tuple unpacking with regular Python expressions, not our syntax.
53 (u"a, b = range(2)", u"a, b = range(2)"),
53 (u"a, b = range(2)", u"a, b = range(2)"),
54 (u"a, = range(1)", u"a, = range(1)"),
54 (u"a, = range(1)", u"a, = range(1)"),
55 (u"a, *bc = range(3)", u"a, *bc = range(3)"),
55 (u"a, *bc = range(3)", u"a, *bc = range(3)"),
56 ]],
56 ]],
57
57
58 assign_magic =
58 assign_magic =
59 [(i,py3compat.u_format(o)) for i,o in \
59 [(i,py3compat.u_format(o)) for i,o in \
60 [(u'a =% who', "a = get_ipython().magic({u}'who')"),
60 [(u'a =% who', "a = get_ipython().magic({u}'who')"),
61 (u'b = %who', "b = get_ipython().magic({u}'who')"),
61 (u'b = %who', "b = get_ipython().magic({u}'who')"),
62 (u'c= %ls', "c = get_ipython().magic({u}'ls')"),
62 (u'c= %ls', "c = get_ipython().magic({u}'ls')"),
63 (u'd == %ls', u'd == %ls'), # Invalid syntax, but we leave == alone.
63 (u'd == %ls', u'd == %ls'), # Invalid syntax, but we leave == alone.
64 ('x=1', 'x=1'), # normal input is unmodified
64 ('x=1', 'x=1'), # normal input is unmodified
65 (' ',' '), # blank lines are kept intact
65 (' ',' '), # blank lines are kept intact
66 (u"a, b = %foo", u"a, b = get_ipython().magic({u}'foo')"),
66 (u"a, b = %foo", u"a, b = get_ipython().magic({u}'foo')"),
67 ]],
67 ]],
68
68
69 classic_prompt =
69 classic_prompt =
70 [('>>> x=1', 'x=1'),
70 [('>>> x=1', 'x=1'),
71 ('x=1', 'x=1'), # normal input is unmodified
71 ('x=1', 'x=1'), # normal input is unmodified
72 (' ', ' '), # blank lines are kept intact
72 (' ', ' '), # blank lines are kept intact
73 ],
73 ],
74
74
75 ipy_prompt =
75 ipy_prompt =
76 [('In [1]: x=1', 'x=1'),
76 [('In [1]: x=1', 'x=1'),
77 ('x=1', 'x=1'), # normal input is unmodified
77 ('x=1', 'x=1'), # normal input is unmodified
78 (' ',' '), # blank lines are kept intact
78 (' ',' '), # blank lines are kept intact
79 ],
79 ],
80
80
81 strip_encoding_cookie =
81 strip_encoding_cookie =
82 [
82 [
83 ('# -*- encoding: utf-8 -*-', ''),
83 ('# -*- encoding: utf-8 -*-', ''),
84 ('# coding: latin-1', ''),
84 ('# coding: latin-1', ''),
85 ],
85 ],
86
86
87
87
88 # Tests for the escape transformer to leave normal code alone
88 # Tests for the escape transformer to leave normal code alone
89 escaped_noesc =
89 escaped_noesc =
90 [ (' ', ' '),
90 [ (' ', ' '),
91 ('x=1', 'x=1'),
91 ('x=1', 'x=1'),
92 ],
92 ],
93
93
94 # System calls
94 # System calls
95 escaped_shell =
95 escaped_shell =
96 [(i,py3compat.u_format(o)) for i,o in \
96 [(i,py3compat.u_format(o)) for i,o in \
97 [ (u'!ls', "get_ipython().system({u}'ls')"),
97 [ (u'!ls', "get_ipython().system({u}'ls')"),
98 # Double-escape shell, this means to capture the output of the
98 # Double-escape shell, this means to capture the output of the
99 # subprocess and return it
99 # subprocess and return it
100 (u'!!ls', "get_ipython().getoutput({u}'ls')"),
100 (u'!!ls', "get_ipython().getoutput({u}'ls')"),
101 ]],
101 ]],
102
102
103 # Help/object info
103 # Help/object info
104 escaped_help =
104 escaped_help =
105 [(i,py3compat.u_format(o)) for i,o in \
105 [(i,py3compat.u_format(o)) for i,o in \
106 [ (u'?', 'get_ipython().show_usage()'),
106 [ (u'?', 'get_ipython().show_usage()'),
107 (u'?x1', "get_ipython().magic({u}'pinfo x1')"),
107 (u'?x1', "get_ipython().magic({u}'pinfo x1')"),
108 (u'??x2', "get_ipython().magic({u}'pinfo2 x2')"),
108 (u'??x2', "get_ipython().magic({u}'pinfo2 x2')"),
109 (u'?a.*s', "get_ipython().magic({u}'psearch a.*s')"),
109 (u'?a.*s', "get_ipython().magic({u}'psearch a.*s')"),
110 (u'?%hist1', "get_ipython().magic({u}'pinfo %hist1')"),
110 (u'?%hist1', "get_ipython().magic({u}'pinfo %hist1')"),
111 (u'?%%hist2', "get_ipython().magic({u}'pinfo %%hist2')"),
111 (u'?%%hist2', "get_ipython().magic({u}'pinfo %%hist2')"),
112 (u'?abc = qwe', "get_ipython().magic({u}'pinfo abc')"),
112 (u'?abc = qwe', "get_ipython().magic({u}'pinfo abc')"),
113 ]],
113 ]],
114
114
115 end_help =
115 end_help =
116 [(i,py3compat.u_format(o)) for i,o in \
116 [(i,py3compat.u_format(o)) for i,o in \
117 [ (u'x3?', "get_ipython().magic({u}'pinfo x3')"),
117 [ (u'x3?', "get_ipython().magic({u}'pinfo x3')"),
118 (u'x4??', "get_ipython().magic({u}'pinfo2 x4')"),
118 (u'x4??', "get_ipython().magic({u}'pinfo2 x4')"),
119 (u'%hist1?', "get_ipython().magic({u}'pinfo %hist1')"),
119 (u'%hist1?', "get_ipython().magic({u}'pinfo %hist1')"),
120 (u'%hist2??', "get_ipython().magic({u}'pinfo2 %hist2')"),
120 (u'%hist2??', "get_ipython().magic({u}'pinfo2 %hist2')"),
121 (u'%%hist3?', "get_ipython().magic({u}'pinfo %%hist3')"),
121 (u'%%hist3?', "get_ipython().magic({u}'pinfo %%hist3')"),
122 (u'%%hist4??', "get_ipython().magic({u}'pinfo2 %%hist4')"),
122 (u'%%hist4??', "get_ipython().magic({u}'pinfo2 %%hist4')"),
123 (u'f*?', "get_ipython().magic({u}'psearch f*')"),
123 (u'f*?', "get_ipython().magic({u}'psearch f*')"),
124 (u'ax.*aspe*?', "get_ipython().magic({u}'psearch ax.*aspe*')"),
124 (u'ax.*aspe*?', "get_ipython().magic({u}'psearch ax.*aspe*')"),
125 (u'a = abc?', "get_ipython().set_next_input({u}'a = abc');"
125 (u'a = abc?', "get_ipython().set_next_input({u}'a = abc');"
126 "get_ipython().magic({u}'pinfo abc')"),
126 "get_ipython().magic({u}'pinfo abc')"),
127 (u'a = abc.qe??', "get_ipython().set_next_input({u}'a = abc.qe');"
127 (u'a = abc.qe??', "get_ipython().set_next_input({u}'a = abc.qe');"
128 "get_ipython().magic({u}'pinfo2 abc.qe')"),
128 "get_ipython().magic({u}'pinfo2 abc.qe')"),
129 (u'a = *.items?', "get_ipython().set_next_input({u}'a = *.items');"
129 (u'a = *.items?', "get_ipython().set_next_input({u}'a = *.items');"
130 "get_ipython().magic({u}'psearch *.items')"),
130 "get_ipython().magic({u}'psearch *.items')"),
131 (u'plot(a?', "get_ipython().set_next_input({u}'plot(a');"
131 (u'plot(a?', "get_ipython().set_next_input({u}'plot(a');"
132 "get_ipython().magic({u}'pinfo a')"),
132 "get_ipython().magic({u}'pinfo a')"),
133 (u'a*2 #comment?', 'a*2 #comment?'),
133 (u'a*2 #comment?', 'a*2 #comment?'),
134 ]],
134 ]],
135
135
136 # Explicit magic calls
136 # Explicit magic calls
137 escaped_magic =
137 escaped_magic =
138 [(i,py3compat.u_format(o)) for i,o in \
138 [(i,py3compat.u_format(o)) for i,o in \
139 [ (u'%cd', "get_ipython().magic({u}'cd')"),
139 [ (u'%cd', "get_ipython().magic({u}'cd')"),
140 (u'%cd /home', "get_ipython().magic({u}'cd /home')"),
140 (u'%cd /home', "get_ipython().magic({u}'cd /home')"),
141 # Backslashes need to be escaped.
141 # Backslashes need to be escaped.
142 (u'%cd C:\\User', "get_ipython().magic({u}'cd C:\\\\User')"),
142 (u'%cd C:\\User', "get_ipython().magic({u}'cd C:\\\\User')"),
143 (u' %magic', " get_ipython().magic({u}'magic')"),
143 (u' %magic', " get_ipython().magic({u}'magic')"),
144 ]],
144 ]],
145
145
146 # Quoting with separate arguments
146 # Quoting with separate arguments
147 escaped_quote =
147 escaped_quote =
148 [ (',f', 'f("")'),
148 [ (',f', 'f("")'),
149 (',f x', 'f("x")'),
149 (',f x', 'f("x")'),
150 (' ,f y', ' f("y")'),
150 (' ,f y', ' f("y")'),
151 (',f a b', 'f("a", "b")'),
151 (',f a b', 'f("a", "b")'),
152 ],
152 ],
153
153
154 # Quoting with single argument
154 # Quoting with single argument
155 escaped_quote2 =
155 escaped_quote2 =
156 [ (';f', 'f("")'),
156 [ (';f', 'f("")'),
157 (';f x', 'f("x")'),
157 (';f x', 'f("x")'),
158 (' ;f y', ' f("y")'),
158 (' ;f y', ' f("y")'),
159 (';f a b', 'f("a b")'),
159 (';f a b', 'f("a b")'),
160 ],
160 ],
161
161
162 # Simply apply parens
162 # Simply apply parens
163 escaped_paren =
163 escaped_paren =
164 [ ('/f', 'f()'),
164 [ ('/f', 'f()'),
165 ('/f x', 'f(x)'),
165 ('/f x', 'f(x)'),
166 (' /f y', ' f(y)'),
166 (' /f y', ' f(y)'),
167 ('/f a b', 'f(a, b)'),
167 ('/f a b', 'f(a, b)'),
168 ],
168 ],
169
169
170 # Check that we transform prompts before other transforms
170 # Check that we transform prompts before other transforms
171 mixed =
171 mixed =
172 [(i,py3compat.u_format(o)) for i,o in \
172 [(i,py3compat.u_format(o)) for i,o in \
173 [ (u'In [1]: %lsmagic', "get_ipython().magic({u}'lsmagic')"),
173 [ (u'In [1]: %lsmagic', "get_ipython().magic({u}'lsmagic')"),
174 (u'>>> %lsmagic', "get_ipython().magic({u}'lsmagic')"),
174 (u'>>> %lsmagic', "get_ipython().magic({u}'lsmagic')"),
175 (u'In [2]: !ls', "get_ipython().system({u}'ls')"),
175 (u'In [2]: !ls', "get_ipython().system({u}'ls')"),
176 (u'In [3]: abs?', "get_ipython().magic({u}'pinfo abs')"),
176 (u'In [3]: abs?', "get_ipython().magic({u}'pinfo abs')"),
177 (u'In [4]: b = %who', "b = get_ipython().magic({u}'who')"),
177 (u'In [4]: b = %who', "b = get_ipython().magic({u}'who')"),
178 ]],
178 ]],
179 )
179 )
180
180
181 # multiline syntax examples. Each of these should be a list of lists, with
181 # multiline syntax examples. Each of these should be a list of lists, with
182 # each entry itself having pairs of raw/transformed input. The union (with
182 # each entry itself having pairs of raw/transformed input. The union (with
183 # '\n'.join() of the transformed inputs is what the splitter should produce
183 # '\n'.join() of the transformed inputs is what the splitter should produce
184 # when fed the raw lines one at a time via push.
184 # when fed the raw lines one at a time via push.
185 syntax_ml = \
185 syntax_ml = \
186 dict(classic_prompt =
186 dict(classic_prompt =
187 [ [('>>> for i in range(10):','for i in range(10):'),
187 [ [('>>> for i in range(10):','for i in range(10):'),
188 ('... print i',' print i'),
188 ('... print i',' print i'),
189 ('... ', ''),
189 ('... ', ''),
190 ],
190 ],
191 [('>>> a="""','a="""'),
191 [('>>> a="""','a="""'),
192 ('... 123"""','123"""'),
192 ('... 123"""','123"""'),
193 ],
193 ],
194 [('a="""','a="""'),
194 [('a="""','a="""'),
195 ('... 123','123'),
195 ('... 123','123'),
196 ('... 456"""','456"""'),
196 ('... 456"""','456"""'),
197 ],
197 ],
198 [('a="""','a="""'),
198 [('a="""','a="""'),
199 ('>>> 123','123'),
199 ('>>> 123','123'),
200 ('... 456"""','456"""'),
200 ('... 456"""','456"""'),
201 ],
201 ],
202 [('a="""','a="""'),
202 [('a="""','a="""'),
203 ('123','123'),
203 ('123','123'),
204 ('... 456"""','... 456"""'),
204 ('... 456"""','... 456"""'),
205 ],
205 ],
206 [('....__class__','....__class__'),
206 [('....__class__','....__class__'),
207 ],
207 ],
208 [('a=5', 'a=5'),
208 [('a=5', 'a=5'),
209 ('...', ''),
209 ('...', ''),
210 ],
210 ],
211 [('>>> def f(x):', 'def f(x):'),
211 [('>>> def f(x):', 'def f(x):'),
212 ('...', ''),
212 ('...', ''),
213 ('... return x', ' return x'),
213 ('... return x', ' return x'),
214 ],
214 ],
215 [('board = """....', 'board = """....'),
215 [('board = """....', 'board = """....'),
216 ('....', '....'),
216 ('....', '....'),
217 ('...."""', '...."""'),
217 ('...."""', '...."""'),
218 ],
218 ],
219 ],
219 ],
220
220
221 ipy_prompt =
221 ipy_prompt =
222 [ [('In [24]: for i in range(10):','for i in range(10):'),
222 [ [('In [24]: for i in range(10):','for i in range(10):'),
223 (' ....: print i',' print i'),
223 (' ....: print i',' print i'),
224 (' ....: ', ''),
224 (' ....: ', ''),
225 ],
225 ],
226 [('In [24]: for i in range(10):','for i in range(10):'),
226 [('In [24]: for i in range(10):','for i in range(10):'),
227 # Qt console prompts expand with spaces, not dots
227 # Qt console prompts expand with spaces, not dots
228 (' ...: print i',' print i'),
228 (' ...: print i',' print i'),
229 (' ...: ', ''),
229 (' ...: ', ''),
230 ],
230 ],
231 [('In [24]: for i in range(10):','for i in range(10):'),
231 [('In [24]: for i in range(10):','for i in range(10):'),
232 # Sometimes whitespace preceding '...' has been removed
232 # Sometimes whitespace preceding '...' has been removed
233 ('...: print i',' print i'),
233 ('...: print i',' print i'),
234 ('...: ', ''),
234 ('...: ', ''),
235 ],
235 ],
236 [('In [24]: for i in range(10):','for i in range(10):'),
236 [('In [24]: for i in range(10):','for i in range(10):'),
237 # Space after last continuation prompt has been removed (issue #6674)
237 # Space after last continuation prompt has been removed (issue #6674)
238 ('...: print i',' print i'),
238 ('...: print i',' print i'),
239 ('...:', ''),
239 ('...:', ''),
240 ],
240 ],
241 [('In [2]: a="""','a="""'),
241 [('In [2]: a="""','a="""'),
242 (' ...: 123"""','123"""'),
242 (' ...: 123"""','123"""'),
243 ],
243 ],
244 [('a="""','a="""'),
244 [('a="""','a="""'),
245 (' ...: 123','123'),
245 (' ...: 123','123'),
246 (' ...: 456"""','456"""'),
246 (' ...: 456"""','456"""'),
247 ],
247 ],
248 [('a="""','a="""'),
248 [('a="""','a="""'),
249 ('In [1]: 123','123'),
249 ('In [1]: 123','123'),
250 (' ...: 456"""','456"""'),
250 (' ...: 456"""','456"""'),
251 ],
251 ],
252 [('a="""','a="""'),
252 [('a="""','a="""'),
253 ('123','123'),
253 ('123','123'),
254 (' ...: 456"""',' ...: 456"""'),
254 (' ...: 456"""',' ...: 456"""'),
255 ],
255 ],
256 ],
256 ],
257
257
258 strip_encoding_cookie =
258 strip_encoding_cookie =
259 [
259 [
260 [
260 [
261 ('# -*- coding: utf-8 -*-', ''),
261 ('# -*- coding: utf-8 -*-', ''),
262 ('foo', 'foo'),
262 ('foo', 'foo'),
263 ],
263 ],
264 [
264 [
265 ('#!/usr/bin/env python', '#!/usr/bin/env python'),
265 ('#!/usr/bin/env python', '#!/usr/bin/env python'),
266 ('# -*- coding: latin-1 -*-', ''),
266 ('# -*- coding: latin-1 -*-', ''),
267 # only the first-two lines
267 # only the first-two lines
268 ('# -*- coding: latin-1 -*-', '# -*- coding: latin-1 -*-'),
268 ('# -*- coding: latin-1 -*-', '# -*- coding: latin-1 -*-'),
269 ],
269 ],
270 ],
270 ],
271
271
272 multiline_datastructure_prompt =
272 multiline_datastructure_prompt =
273 [ [('>>> a = [1,','a = [1,'),
273 [ [('>>> a = [1,','a = [1,'),
274 ('... 2]','2]'),
274 ('... 2]','2]'),
275 ],
275 ],
276 ],
276 ],
277
277
278 multiline_datastructure =
278 multiline_datastructure =
279 [ [('b = ("%s"', None),
279 [ [('b = ("%s"', None),
280 ('# comment', None),
280 ('# comment', None),
281 ('%foo )', 'b = ("%s"\n# comment\n%foo )'),
281 ('%foo )', 'b = ("%s"\n# comment\n%foo )'),
282 ],
282 ],
283 ],
283 ],
284
284
285 multiline_string =
285 multiline_string =
286 [ [("'''foo?", None),
286 [ [("'''foo?", None),
287 ("bar'''", "'''foo?\nbar'''"),
287 ("bar'''", "'''foo?\nbar'''"),
288 ],
288 ],
289 ],
289 ],
290
290
291 leading_indent =
291 leading_indent =
292 [ [(' print "hi"','print "hi"'),
292 [ [(' print "hi"','print "hi"'),
293 ],
293 ],
294 [(' for a in range(5):','for a in range(5):'),
294 [(' for a in range(5):','for a in range(5):'),
295 (' a*2',' a*2'),
295 (' a*2',' a*2'),
296 ],
296 ],
297 [(' a="""','a="""'),
297 [(' a="""','a="""'),
298 (' 123"""','123"""'),
298 (' 123"""','123"""'),
299 ],
299 ],
300 [('a="""','a="""'),
300 [('a="""','a="""'),
301 (' 123"""',' 123"""'),
301 (' 123"""',' 123"""'),
302 ],
302 ],
303 ],
303 ],
304
304
305 cellmagic =
305 cellmagic =
306 [ [(u'%%foo a', None),
306 [ [(u'%%foo a', None),
307 (None, u_fmt("get_ipython().run_cell_magic({u}'foo', {u}'a', {u}'')")),
307 (None, u_fmt("get_ipython().run_cell_magic({u}'foo', {u}'a', {u}'')")),
308 ],
308 ],
309 [(u'%%bar 123', None),
309 [(u'%%bar 123', None),
310 (u'hello', None),
310 (u'hello', None),
311 (None , u_fmt("get_ipython().run_cell_magic({u}'bar', {u}'123', {u}'hello')")),
311 (None , u_fmt("get_ipython().run_cell_magic({u}'bar', {u}'123', {u}'hello')")),
312 ],
312 ],
313 [(u'a=5', 'a=5'),
313 [(u'a=5', 'a=5'),
314 (u'%%cellmagic', '%%cellmagic'),
314 (u'%%cellmagic', '%%cellmagic'),
315 ],
315 ],
316 ],
316 ],
317
317
318 escaped =
318 escaped =
319 [ [('%abc def \\', None),
319 [ [('%abc def \\', None),
320 ('ghi', u_fmt("get_ipython().magic({u}'abc def ghi')")),
320 ('ghi', u_fmt("get_ipython().magic({u}'abc def ghi')")),
321 ],
321 ],
322 [('%abc def \\', None),
322 [('%abc def \\', None),
323 ('ghi\\', None),
323 ('ghi\\', None),
324 (None, u_fmt("get_ipython().magic({u}'abc def ghi')")),
324 (None, u_fmt("get_ipython().magic({u}'abc def ghi')")),
325 ],
325 ],
326 ],
326 ],
327
327
328 assign_magic =
328 assign_magic =
329 [ [(u'a = %bc de \\', None),
329 [ [(u'a = %bc de \\', None),
330 (u'fg', u_fmt("a = get_ipython().magic({u}'bc de fg')")),
330 (u'fg', u_fmt("a = get_ipython().magic({u}'bc de fg')")),
331 ],
331 ],
332 [(u'a = %bc de \\', None),
332 [(u'a = %bc de \\', None),
333 (u'fg\\', None),
333 (u'fg\\', None),
334 (None, u_fmt("a = get_ipython().magic({u}'bc de fg')")),
334 (None, u_fmt("a = get_ipython().magic({u}'bc de fg')")),
335 ],
335 ],
336 ],
336 ],
337
337
338 assign_system =
338 assign_system =
339 [ [(u'a = !bc de \\', None),
339 [ [(u'a = !bc de \\', None),
340 (u'fg', u_fmt("a = get_ipython().getoutput({u}'bc de fg')")),
340 (u'fg', u_fmt("a = get_ipython().getoutput({u}'bc de fg')")),
341 ],
341 ],
342 [(u'a = !bc de \\', None),
342 [(u'a = !bc de \\', None),
343 (u'fg\\', None),
343 (u'fg\\', None),
344 (None, u_fmt("a = get_ipython().getoutput({u}'bc de fg')")),
344 (None, u_fmt("a = get_ipython().getoutput({u}'bc de fg')")),
345 ],
345 ],
346 ],
346 ],
347 )
347 )
348
348
349
349
350 def test_assign_system():
350 def test_assign_system():
351 tt.check_pairs(transform_and_reset(ipt.assign_from_system), syntax['assign_system'])
351 tt.check_pairs(transform_and_reset(ipt.assign_from_system), syntax['assign_system'])
352
352
353 def test_assign_magic():
353 def test_assign_magic():
354 tt.check_pairs(transform_and_reset(ipt.assign_from_magic), syntax['assign_magic'])
354 tt.check_pairs(transform_and_reset(ipt.assign_from_magic), syntax['assign_magic'])
355
355
356 def test_classic_prompt():
356 def test_classic_prompt():
357 tt.check_pairs(transform_and_reset(ipt.classic_prompt), syntax['classic_prompt'])
357 tt.check_pairs(transform_and_reset(ipt.classic_prompt), syntax['classic_prompt'])
358 for example in syntax_ml['classic_prompt']:
358 for example in syntax_ml['classic_prompt']:
359 transform_checker(example, ipt.classic_prompt)
359 transform_checker(example, ipt.classic_prompt)
360 for example in syntax_ml['multiline_datastructure_prompt']:
360 for example in syntax_ml['multiline_datastructure_prompt']:
361 transform_checker(example, ipt.classic_prompt)
361 transform_checker(example, ipt.classic_prompt)
362
362
363 # Check that we don't transform the second line if the first is obviously
363 # Check that we don't transform the second line if the first is obviously
364 # IPython syntax
364 # IPython syntax
365 transform_checker([
365 transform_checker([
366 (u'%foo', '%foo'),
366 (u'%foo', '%foo'),
367 (u'>>> bar', '>>> bar'),
367 (u'>>> bar', '>>> bar'),
368 ], ipt.classic_prompt)
368 ], ipt.classic_prompt)
369
369
370
370
371 def test_ipy_prompt():
371 def test_ipy_prompt():
372 tt.check_pairs(transform_and_reset(ipt.ipy_prompt), syntax['ipy_prompt'])
372 tt.check_pairs(transform_and_reset(ipt.ipy_prompt), syntax['ipy_prompt'])
373 for example in syntax_ml['ipy_prompt']:
373 for example in syntax_ml['ipy_prompt']:
374 transform_checker(example, ipt.ipy_prompt)
374 transform_checker(example, ipt.ipy_prompt)
375
375
376 # Check that we don't transform the second line if we're inside a cell magic
376 # Check that we don't transform the second line if we're inside a cell magic
377 transform_checker([
377 transform_checker([
378 (u'%%foo', '%%foo'),
378 (u'%%foo', '%%foo'),
379 (u'In [1]: bar', 'In [1]: bar'),
379 (u'In [1]: bar', 'In [1]: bar'),
380 ], ipt.ipy_prompt)
380 ], ipt.ipy_prompt)
381
381
382 def test_coding_cookie():
382 def test_coding_cookie():
383 tt.check_pairs(transform_and_reset(ipt.strip_encoding_cookie), syntax['strip_encoding_cookie'])
383 tt.check_pairs(transform_and_reset(ipt.strip_encoding_cookie), syntax['strip_encoding_cookie'])
384 for example in syntax_ml['strip_encoding_cookie']:
384 for example in syntax_ml['strip_encoding_cookie']:
385 transform_checker(example, ipt.strip_encoding_cookie)
385 transform_checker(example, ipt.strip_encoding_cookie)
386
386
387 def test_assemble_logical_lines():
387 def test_assemble_logical_lines():
388 tests = \
388 tests = \
389 [ [(u"a = \\", None),
389 [ [(u"a = \\", None),
390 (u"123", u"a = 123"),
390 (u"123", u"a = 123"),
391 ],
391 ],
392 [(u"a = \\", None), # Test resetting when within a multi-line string
392 [(u"a = \\", None), # Test resetting when within a multi-line string
393 (u"12 *\\", None),
393 (u"12 *\\", None),
394 (None, u"a = 12 *"),
394 (None, u"a = 12 *"),
395 ],
395 ],
396 [(u"# foo\\", u"# foo\\"), # Comments can't be continued like this
396 [(u"# foo\\", u"# foo\\"), # Comments can't be continued like this
397 ],
397 ],
398 ]
398 ]
399 for example in tests:
399 for example in tests:
400 transform_checker(example, ipt.assemble_logical_lines)
400 transform_checker(example, ipt.assemble_logical_lines)
401
401
402 def test_assemble_python_lines():
402 def test_assemble_python_lines():
403 tests = \
403 tests = \
404 [ [(u"a = '''", None),
404 [ [(u"a = '''", None),
405 (u"abc'''", u"a = '''\nabc'''"),
405 (u"abc'''", u"a = '''\nabc'''"),
406 ],
406 ],
407 [(u"a = '''", None), # Test resetting when within a multi-line string
407 [(u"a = '''", None), # Test resetting when within a multi-line string
408 (u"def", None),
408 (u"def", None),
409 (None, u"a = '''\ndef"),
409 (None, u"a = '''\ndef"),
410 ],
410 ],
411 [(u"a = [1,", None),
411 [(u"a = [1,", None),
412 (u"2]", u"a = [1,\n2]"),
412 (u"2]", u"a = [1,\n2]"),
413 ],
413 ],
414 [(u"a = [1,", None), # Test resetting when within a multi-line string
414 [(u"a = [1,", None), # Test resetting when within a multi-line string
415 (u"2,", None),
415 (u"2,", None),
416 (None, u"a = [1,\n2,"),
416 (None, u"a = [1,\n2,"),
417 ],
417 ],
418 [(u"a = '''", None), # Test line continuation within a multi-line string
419 (u"abc\\", None),
420 (u"def", None),
421 (u"'''", u"a = '''\nabc\\\ndef\n'''"),
422 ],
418 ] + syntax_ml['multiline_datastructure']
423 ] + syntax_ml['multiline_datastructure']
419 for example in tests:
424 for example in tests:
420 transform_checker(example, ipt.assemble_python_lines)
425 transform_checker(example, ipt.assemble_python_lines)
421
426
422
427
423 def test_help_end():
428 def test_help_end():
424 tt.check_pairs(transform_and_reset(ipt.help_end), syntax['end_help'])
429 tt.check_pairs(transform_and_reset(ipt.help_end), syntax['end_help'])
425
430
426 def test_escaped_noesc():
431 def test_escaped_noesc():
427 tt.check_pairs(transform_and_reset(ipt.escaped_commands), syntax['escaped_noesc'])
432 tt.check_pairs(transform_and_reset(ipt.escaped_commands), syntax['escaped_noesc'])
428
433
429
434
430 def test_escaped_shell():
435 def test_escaped_shell():
431 tt.check_pairs(transform_and_reset(ipt.escaped_commands), syntax['escaped_shell'])
436 tt.check_pairs(transform_and_reset(ipt.escaped_commands), syntax['escaped_shell'])
432
437
433
438
434 def test_escaped_help():
439 def test_escaped_help():
435 tt.check_pairs(transform_and_reset(ipt.escaped_commands), syntax['escaped_help'])
440 tt.check_pairs(transform_and_reset(ipt.escaped_commands), syntax['escaped_help'])
436
441
437
442
438 def test_escaped_magic():
443 def test_escaped_magic():
439 tt.check_pairs(transform_and_reset(ipt.escaped_commands), syntax['escaped_magic'])
444 tt.check_pairs(transform_and_reset(ipt.escaped_commands), syntax['escaped_magic'])
440
445
441
446
442 def test_escaped_quote():
447 def test_escaped_quote():
443 tt.check_pairs(transform_and_reset(ipt.escaped_commands), syntax['escaped_quote'])
448 tt.check_pairs(transform_and_reset(ipt.escaped_commands), syntax['escaped_quote'])
444
449
445
450
446 def test_escaped_quote2():
451 def test_escaped_quote2():
447 tt.check_pairs(transform_and_reset(ipt.escaped_commands), syntax['escaped_quote2'])
452 tt.check_pairs(transform_and_reset(ipt.escaped_commands), syntax['escaped_quote2'])
448
453
449
454
450 def test_escaped_paren():
455 def test_escaped_paren():
451 tt.check_pairs(transform_and_reset(ipt.escaped_commands), syntax['escaped_paren'])
456 tt.check_pairs(transform_and_reset(ipt.escaped_commands), syntax['escaped_paren'])
452
457
453
458
454 def test_cellmagic():
459 def test_cellmagic():
455 for example in syntax_ml['cellmagic']:
460 for example in syntax_ml['cellmagic']:
456 transform_checker(example, ipt.cellmagic)
461 transform_checker(example, ipt.cellmagic)
457
462
458 line_example = [(u'%%bar 123', None),
463 line_example = [(u'%%bar 123', None),
459 (u'hello', None),
464 (u'hello', None),
460 (u'' , u_fmt("get_ipython().run_cell_magic({u}'bar', {u}'123', {u}'hello')")),
465 (u'' , u_fmt("get_ipython().run_cell_magic({u}'bar', {u}'123', {u}'hello')")),
461 ]
466 ]
462 transform_checker(line_example, ipt.cellmagic, end_on_blank_line=True)
467 transform_checker(line_example, ipt.cellmagic, end_on_blank_line=True)
463
468
464 def test_has_comment():
469 def test_has_comment():
465 tests = [('text', False),
470 tests = [('text', False),
466 ('text #comment', True),
471 ('text #comment', True),
467 ('text #comment\n', True),
472 ('text #comment\n', True),
468 ('#comment', True),
473 ('#comment', True),
469 ('#comment\n', True),
474 ('#comment\n', True),
470 ('a = "#string"', False),
475 ('a = "#string"', False),
471 ('a = "#string" # comment', True),
476 ('a = "#string" # comment', True),
472 ('a #comment not "string"', True),
477 ('a #comment not "string"', True),
473 ]
478 ]
474 tt.check_pairs(ipt.has_comment, tests)
479 tt.check_pairs(ipt.has_comment, tests)
475
480
476 @ipt.TokenInputTransformer.wrap
481 @ipt.TokenInputTransformer.wrap
477 def decistmt(tokens):
482 def decistmt(tokens):
478 """Substitute Decimals for floats in a string of statements.
483 """Substitute Decimals for floats in a string of statements.
479
484
480 Based on an example from the tokenize module docs.
485 Based on an example from the tokenize module docs.
481 """
486 """
482 result = []
487 result = []
483 for toknum, tokval, _, _, _ in tokens:
488 for toknum, tokval, _, _, _ in tokens:
484 if toknum == tokenize.NUMBER and '.' in tokval: # replace NUMBER tokens
489 if toknum == tokenize.NUMBER and '.' in tokval: # replace NUMBER tokens
485 for newtok in [
490 for newtok in [
486 (tokenize.NAME, 'Decimal'),
491 (tokenize.NAME, 'Decimal'),
487 (tokenize.OP, '('),
492 (tokenize.OP, '('),
488 (tokenize.STRING, repr(tokval)),
493 (tokenize.STRING, repr(tokval)),
489 (tokenize.OP, ')')
494 (tokenize.OP, ')')
490 ]:
495 ]:
491 yield newtok
496 yield newtok
492 else:
497 else:
493 yield (toknum, tokval)
498 yield (toknum, tokval)
494
499
495
500
496
501
497 def test_token_input_transformer():
502 def test_token_input_transformer():
498 tests = [(u'1.2', u_fmt(u"Decimal ({u}'1.2')")),
503 tests = [(u'1.2', u_fmt(u"Decimal ({u}'1.2')")),
499 (u'"1.2"', u'"1.2"'),
504 (u'"1.2"', u'"1.2"'),
500 ]
505 ]
501 tt.check_pairs(transform_and_reset(decistmt), tests)
506 tt.check_pairs(transform_and_reset(decistmt), tests)
502 ml_tests = \
507 ml_tests = \
503 [ [(u"a = 1.2; b = '''x", None),
508 [ [(u"a = 1.2; b = '''x", None),
504 (u"y'''", u_fmt(u"a =Decimal ({u}'1.2');b ='''x\ny'''")),
509 (u"y'''", u_fmt(u"a =Decimal ({u}'1.2');b ='''x\ny'''")),
505 ],
510 ],
506 [(u"a = [1.2,", None),
511 [(u"a = [1.2,", None),
507 (u"3]", u_fmt(u"a =[Decimal ({u}'1.2'),\n3 ]")),
512 (u"3]", u_fmt(u"a =[Decimal ({u}'1.2'),\n3 ]")),
508 ],
513 ],
509 [(u"a = '''foo", None), # Test resetting when within a multi-line string
514 [(u"a = '''foo", None), # Test resetting when within a multi-line string
510 (u"bar", None),
515 (u"bar", None),
511 (None, u"a = '''foo\nbar"),
516 (None, u"a = '''foo\nbar"),
512 ],
517 ],
513 ]
518 ]
514 for example in ml_tests:
519 for example in ml_tests:
515 transform_checker(example, decistmt)
520 transform_checker(example, decistmt)
General Comments 0
You need to be logged in to leave comments. Login now