##// END OF EJS Templates
Merge pull request #11398 from Carreau/fix-input-docstring...
Matthias Bussonnier -
r24702:0f1de669 merge
parent child Browse files
Show More
@@ -1,694 +1,699 b''
1 """Input transformer machinery to support IPython special syntax.
1 """Input transformer machinery to support IPython special syntax.
2
2
3 This includes the machinery to recognise and transform ``%magic`` commands,
3 This includes the machinery to recognise and transform ``%magic`` commands,
4 ``!system`` commands, ``help?`` querying, prompt stripping, and so forth.
4 ``!system`` commands, ``help?`` querying, prompt stripping, and so forth.
5
5
6 Added: IPython 7.0. Replaces inputsplitter and inputtransformer which were
6 Added: IPython 7.0. Replaces inputsplitter and inputtransformer which were
7 deprecated in 7.0.
7 deprecated in 7.0.
8 """
8 """
9
9
10 # Copyright (c) IPython Development Team.
10 # Copyright (c) IPython Development Team.
11 # Distributed under the terms of the Modified BSD License.
11 # Distributed under the terms of the Modified BSD License.
12
12
13 from codeop import compile_command
13 from codeop import compile_command
14 import re
14 import re
15 import tokenize
15 import tokenize
16 from typing import List, Tuple
16 from typing import List, Tuple
17 import warnings
17 import warnings
18
18
19 _indent_re = re.compile(r'^[ \t]+')
19 _indent_re = re.compile(r'^[ \t]+')
20
20
21 def leading_indent(lines):
21 def leading_indent(lines):
22 """Remove leading indentation.
22 """Remove leading indentation.
23
23
24 If the first line starts with a spaces or tabs, the same whitespace will be
24 If the first line starts with a spaces or tabs, the same whitespace will be
25 removed from each following line in the cell.
25 removed from each following line in the cell.
26 """
26 """
27 if not lines:
27 if not lines:
28 return lines
28 return lines
29 m = _indent_re.match(lines[0])
29 m = _indent_re.match(lines[0])
30 if not m:
30 if not m:
31 return lines
31 return lines
32 space = m.group(0)
32 space = m.group(0)
33 n = len(space)
33 n = len(space)
34 return [l[n:] if l.startswith(space) else l
34 return [l[n:] if l.startswith(space) else l
35 for l in lines]
35 for l in lines]
36
36
37 class PromptStripper:
37 class PromptStripper:
38 """Remove matching input prompts from a block of input.
38 """Remove matching input prompts from a block of input.
39
39
40 Parameters
40 Parameters
41 ----------
41 ----------
42 prompt_re : regular expression
42 prompt_re : regular expression
43 A regular expression matching any input prompt (including continuation,
43 A regular expression matching any input prompt (including continuation,
44 e.g. ``...``)
44 e.g. ``...``)
45 initial_re : regular expression, optional
45 initial_re : regular expression, optional
46 A regular expression matching only the initial prompt, but not continuation.
46 A regular expression matching only the initial prompt, but not continuation.
47 If no initial expression is given, prompt_re will be used everywhere.
47 If no initial expression is given, prompt_re will be used everywhere.
48 Used mainly for plain Python prompts (``>>>``), where the continuation prompt
48 Used mainly for plain Python prompts (``>>>``), where the continuation prompt
49 ``...`` is a valid Python expression in Python 3, so shouldn't be stripped.
49 ``...`` is a valid Python expression in Python 3, so shouldn't be stripped.
50
50
51 If initial_re and prompt_re differ,
51 If initial_re and prompt_re differ,
52 only initial_re will be tested against the first line.
52 only initial_re will be tested against the first line.
53 If any prompt is found on the first two lines,
53 If any prompt is found on the first two lines,
54 prompts will be stripped from the rest of the block.
54 prompts will be stripped from the rest of the block.
55 """
55 """
56 def __init__(self, prompt_re, initial_re=None):
56 def __init__(self, prompt_re, initial_re=None):
57 self.prompt_re = prompt_re
57 self.prompt_re = prompt_re
58 self.initial_re = initial_re or prompt_re
58 self.initial_re = initial_re or prompt_re
59
59
60 def _strip(self, lines):
60 def _strip(self, lines):
61 return [self.prompt_re.sub('', l, count=1) for l in lines]
61 return [self.prompt_re.sub('', l, count=1) for l in lines]
62
62
63 def __call__(self, lines):
63 def __call__(self, lines):
64 if not lines:
64 if not lines:
65 return lines
65 return lines
66 if self.initial_re.match(lines[0]) or \
66 if self.initial_re.match(lines[0]) or \
67 (len(lines) > 1 and self.prompt_re.match(lines[1])):
67 (len(lines) > 1 and self.prompt_re.match(lines[1])):
68 return self._strip(lines)
68 return self._strip(lines)
69 return lines
69 return lines
70
70
71 classic_prompt = PromptStripper(
71 classic_prompt = PromptStripper(
72 prompt_re=re.compile(r'^(>>>|\.\.\.)( |$)'),
72 prompt_re=re.compile(r'^(>>>|\.\.\.)( |$)'),
73 initial_re=re.compile(r'^>>>( |$)')
73 initial_re=re.compile(r'^>>>( |$)')
74 )
74 )
75
75
76 ipython_prompt = PromptStripper(re.compile(r'^(In \[\d+\]: |\s*\.{3,}: ?)'))
76 ipython_prompt = PromptStripper(re.compile(r'^(In \[\d+\]: |\s*\.{3,}: ?)'))
77
77
78 def cell_magic(lines):
78 def cell_magic(lines):
79 if not lines or not lines[0].startswith('%%'):
79 if not lines or not lines[0].startswith('%%'):
80 return lines
80 return lines
81 if re.match('%%\w+\?', lines[0]):
81 if re.match('%%\w+\?', lines[0]):
82 # This case will be handled by help_end
82 # This case will be handled by help_end
83 return lines
83 return lines
84 magic_name, _, first_line = lines[0][2:-1].partition(' ')
84 magic_name, _, first_line = lines[0][2:-1].partition(' ')
85 body = ''.join(lines[1:])
85 body = ''.join(lines[1:])
86 return ['get_ipython().run_cell_magic(%r, %r, %r)\n'
86 return ['get_ipython().run_cell_magic(%r, %r, %r)\n'
87 % (magic_name, first_line, body)]
87 % (magic_name, first_line, body)]
88
88
89
89
90 def _find_assign_op(token_line):
90 def _find_assign_op(token_line):
91 """Get the index of the first assignment in the line ('=' not inside brackets)
91 """Get the index of the first assignment in the line ('=' not inside brackets)
92
92
93 Note: We don't try to support multiple special assignment (a = b = %foo)
93 Note: We don't try to support multiple special assignment (a = b = %foo)
94 """
94 """
95 paren_level = 0
95 paren_level = 0
96 for i, ti in enumerate(token_line):
96 for i, ti in enumerate(token_line):
97 s = ti.string
97 s = ti.string
98 if s == '=' and paren_level == 0:
98 if s == '=' and paren_level == 0:
99 return i
99 return i
100 if s in '([{':
100 if s in '([{':
101 paren_level += 1
101 paren_level += 1
102 elif s in ')]}':
102 elif s in ')]}':
103 if paren_level > 0:
103 if paren_level > 0:
104 paren_level -= 1
104 paren_level -= 1
105
105
106 def find_end_of_continued_line(lines, start_line: int):
106 def find_end_of_continued_line(lines, start_line: int):
107 """Find the last line of a line explicitly extended using backslashes.
107 """Find the last line of a line explicitly extended using backslashes.
108
108
109 Uses 0-indexed line numbers.
109 Uses 0-indexed line numbers.
110 """
110 """
111 end_line = start_line
111 end_line = start_line
112 while lines[end_line].endswith('\\\n'):
112 while lines[end_line].endswith('\\\n'):
113 end_line += 1
113 end_line += 1
114 if end_line >= len(lines):
114 if end_line >= len(lines):
115 break
115 break
116 return end_line
116 return end_line
117
117
118 def assemble_continued_line(lines, start: Tuple[int, int], end_line: int):
118 def assemble_continued_line(lines, start: Tuple[int, int], end_line: int):
119 """Assemble a single line from multiple continued line pieces
119 """Assemble a single line from multiple continued line pieces
120
120
121 Continued lines are lines ending in ``\``, and the line following the last
121 Continued lines are lines ending in ``\``, and the line following the last
122 ``\`` in the block.
122 ``\`` in the block.
123
123
124 For example, this code continues over multiple lines::
124 For example, this code continues over multiple lines::
125
125
126 if (assign_ix is not None) \
126 if (assign_ix is not None) \
127 and (len(line) >= assign_ix + 2) \
127 and (len(line) >= assign_ix + 2) \
128 and (line[assign_ix+1].string == '%') \
128 and (line[assign_ix+1].string == '%') \
129 and (line[assign_ix+2].type == tokenize.NAME):
129 and (line[assign_ix+2].type == tokenize.NAME):
130
130
131 This statement contains four continued line pieces.
131 This statement contains four continued line pieces.
132 Assembling these pieces into a single line would give::
132 Assembling these pieces into a single line would give::
133
133
134 if (assign_ix is not None) and (len(line) >= assign_ix + 2) and (line[...
134 if (assign_ix is not None) and (len(line) >= assign_ix + 2) and (line[...
135
135
136 This uses 0-indexed line numbers. *start* is (lineno, colno).
136 This uses 0-indexed line numbers. *start* is (lineno, colno).
137
137
138 Used to allow ``%magic`` and ``!system`` commands to be continued over
138 Used to allow ``%magic`` and ``!system`` commands to be continued over
139 multiple lines.
139 multiple lines.
140 """
140 """
141 parts = [lines[start[0]][start[1]:]] + lines[start[0]+1:end_line+1]
141 parts = [lines[start[0]][start[1]:]] + lines[start[0]+1:end_line+1]
142 return ' '.join([p[:-2] for p in parts[:-1]] # Strip backslash+newline
142 return ' '.join([p[:-2] for p in parts[:-1]] # Strip backslash+newline
143 + [parts[-1][:-1]]) # Strip newline from last line
143 + [parts[-1][:-1]]) # Strip newline from last line
144
144
145 class TokenTransformBase:
145 class TokenTransformBase:
146 """Base class for transformations which examine tokens.
146 """Base class for transformations which examine tokens.
147
147
148 Special syntax should not be transformed when it occurs inside strings or
148 Special syntax should not be transformed when it occurs inside strings or
149 comments. This is hard to reliably avoid with regexes. The solution is to
149 comments. This is hard to reliably avoid with regexes. The solution is to
150 tokenise the code as Python, and recognise the special syntax in the tokens.
150 tokenise the code as Python, and recognise the special syntax in the tokens.
151
151
152 IPython's special syntax is not valid Python syntax, so tokenising may go
152 IPython's special syntax is not valid Python syntax, so tokenising may go
153 wrong after the special syntax starts. These classes therefore find and
153 wrong after the special syntax starts. These classes therefore find and
154 transform *one* instance of special syntax at a time into regular Python
154 transform *one* instance of special syntax at a time into regular Python
155 syntax. After each transformation, tokens are regenerated to find the next
155 syntax. After each transformation, tokens are regenerated to find the next
156 piece of special syntax.
156 piece of special syntax.
157
157
158 Subclasses need to implement one class method (find)
158 Subclasses need to implement one class method (find)
159 and one regular method (transform).
159 and one regular method (transform).
160
160
161 The priority attribute can select which transformation to apply if multiple
161 The priority attribute can select which transformation to apply if multiple
162 transformers match in the same place. Lower numbers have higher priority.
162 transformers match in the same place. Lower numbers have higher priority.
163 This allows "%magic?" to be turned into a help call rather than a magic call.
163 This allows "%magic?" to be turned into a help call rather than a magic call.
164 """
164 """
165 # Lower numbers -> higher priority (for matches in the same location)
165 # Lower numbers -> higher priority (for matches in the same location)
166 priority = 10
166 priority = 10
167
167
168 def sortby(self):
168 def sortby(self):
169 return self.start_line, self.start_col, self.priority
169 return self.start_line, self.start_col, self.priority
170
170
171 def __init__(self, start):
171 def __init__(self, start):
172 self.start_line = start[0] - 1 # Shift from 1-index to 0-index
172 self.start_line = start[0] - 1 # Shift from 1-index to 0-index
173 self.start_col = start[1]
173 self.start_col = start[1]
174
174
175 @classmethod
175 @classmethod
176 def find(cls, tokens_by_line):
176 def find(cls, tokens_by_line):
177 """Find one instance of special syntax in the provided tokens.
177 """Find one instance of special syntax in the provided tokens.
178
178
179 Tokens are grouped into logical lines for convenience,
179 Tokens are grouped into logical lines for convenience,
180 so it is easy to e.g. look at the first token of each line.
180 so it is easy to e.g. look at the first token of each line.
181 *tokens_by_line* is a list of lists of tokenize.TokenInfo objects.
181 *tokens_by_line* is a list of lists of tokenize.TokenInfo objects.
182
182
183 This should return an instance of its class, pointing to the start
183 This should return an instance of its class, pointing to the start
184 position it has found, or None if it found no match.
184 position it has found, or None if it found no match.
185 """
185 """
186 raise NotImplementedError
186 raise NotImplementedError
187
187
188 def transform(self, lines: List[str]):
188 def transform(self, lines: List[str]):
189 """Transform one instance of special syntax found by ``find()``
189 """Transform one instance of special syntax found by ``find()``
190
190
191 Takes a list of strings representing physical lines,
191 Takes a list of strings representing physical lines,
192 returns a similar list of transformed lines.
192 returns a similar list of transformed lines.
193 """
193 """
194 raise NotImplementedError
194 raise NotImplementedError
195
195
196 class MagicAssign(TokenTransformBase):
196 class MagicAssign(TokenTransformBase):
197 """Transformer for assignments from magics (a = %foo)"""
197 """Transformer for assignments from magics (a = %foo)"""
198 @classmethod
198 @classmethod
199 def find(cls, tokens_by_line):
199 def find(cls, tokens_by_line):
200 """Find the first magic assignment (a = %foo) in the cell.
200 """Find the first magic assignment (a = %foo) in the cell.
201 """
201 """
202 for line in tokens_by_line:
202 for line in tokens_by_line:
203 assign_ix = _find_assign_op(line)
203 assign_ix = _find_assign_op(line)
204 if (assign_ix is not None) \
204 if (assign_ix is not None) \
205 and (len(line) >= assign_ix + 2) \
205 and (len(line) >= assign_ix + 2) \
206 and (line[assign_ix+1].string == '%') \
206 and (line[assign_ix+1].string == '%') \
207 and (line[assign_ix+2].type == tokenize.NAME):
207 and (line[assign_ix+2].type == tokenize.NAME):
208 return cls(line[assign_ix+1].start)
208 return cls(line[assign_ix+1].start)
209
209
210 def transform(self, lines: List[str]):
210 def transform(self, lines: List[str]):
211 """Transform a magic assignment found by the ``find()`` classmethod.
211 """Transform a magic assignment found by the ``find()`` classmethod.
212 """
212 """
213 start_line, start_col = self.start_line, self.start_col
213 start_line, start_col = self.start_line, self.start_col
214 lhs = lines[start_line][:start_col]
214 lhs = lines[start_line][:start_col]
215 end_line = find_end_of_continued_line(lines, start_line)
215 end_line = find_end_of_continued_line(lines, start_line)
216 rhs = assemble_continued_line(lines, (start_line, start_col), end_line)
216 rhs = assemble_continued_line(lines, (start_line, start_col), end_line)
217 assert rhs.startswith('%'), rhs
217 assert rhs.startswith('%'), rhs
218 magic_name, _, args = rhs[1:].partition(' ')
218 magic_name, _, args = rhs[1:].partition(' ')
219
219
220 lines_before = lines[:start_line]
220 lines_before = lines[:start_line]
221 call = "get_ipython().run_line_magic({!r}, {!r})".format(magic_name, args)
221 call = "get_ipython().run_line_magic({!r}, {!r})".format(magic_name, args)
222 new_line = lhs + call + '\n'
222 new_line = lhs + call + '\n'
223 lines_after = lines[end_line+1:]
223 lines_after = lines[end_line+1:]
224
224
225 return lines_before + [new_line] + lines_after
225 return lines_before + [new_line] + lines_after
226
226
227
227
228 class SystemAssign(TokenTransformBase):
228 class SystemAssign(TokenTransformBase):
229 """Transformer for assignments from system commands (a = !foo)"""
229 """Transformer for assignments from system commands (a = !foo)"""
230 @classmethod
230 @classmethod
231 def find(cls, tokens_by_line):
231 def find(cls, tokens_by_line):
232 """Find the first system assignment (a = !foo) in the cell.
232 """Find the first system assignment (a = !foo) in the cell.
233 """
233 """
234 for line in tokens_by_line:
234 for line in tokens_by_line:
235 assign_ix = _find_assign_op(line)
235 assign_ix = _find_assign_op(line)
236 if (assign_ix is not None) \
236 if (assign_ix is not None) \
237 and not line[assign_ix].line.strip().startswith('=') \
237 and not line[assign_ix].line.strip().startswith('=') \
238 and (len(line) >= assign_ix + 2) \
238 and (len(line) >= assign_ix + 2) \
239 and (line[assign_ix + 1].type == tokenize.ERRORTOKEN):
239 and (line[assign_ix + 1].type == tokenize.ERRORTOKEN):
240 ix = assign_ix + 1
240 ix = assign_ix + 1
241
241
242 while ix < len(line) and line[ix].type == tokenize.ERRORTOKEN:
242 while ix < len(line) and line[ix].type == tokenize.ERRORTOKEN:
243 if line[ix].string == '!':
243 if line[ix].string == '!':
244 return cls(line[ix].start)
244 return cls(line[ix].start)
245 elif not line[ix].string.isspace():
245 elif not line[ix].string.isspace():
246 break
246 break
247 ix += 1
247 ix += 1
248
248
249 def transform(self, lines: List[str]):
249 def transform(self, lines: List[str]):
250 """Transform a system assignment found by the ``find()`` classmethod.
250 """Transform a system assignment found by the ``find()`` classmethod.
251 """
251 """
252 start_line, start_col = self.start_line, self.start_col
252 start_line, start_col = self.start_line, self.start_col
253
253
254 lhs = lines[start_line][:start_col]
254 lhs = lines[start_line][:start_col]
255 end_line = find_end_of_continued_line(lines, start_line)
255 end_line = find_end_of_continued_line(lines, start_line)
256 rhs = assemble_continued_line(lines, (start_line, start_col), end_line)
256 rhs = assemble_continued_line(lines, (start_line, start_col), end_line)
257 assert rhs.startswith('!'), rhs
257 assert rhs.startswith('!'), rhs
258 cmd = rhs[1:]
258 cmd = rhs[1:]
259
259
260 lines_before = lines[:start_line]
260 lines_before = lines[:start_line]
261 call = "get_ipython().getoutput({!r})".format(cmd)
261 call = "get_ipython().getoutput({!r})".format(cmd)
262 new_line = lhs + call + '\n'
262 new_line = lhs + call + '\n'
263 lines_after = lines[end_line + 1:]
263 lines_after = lines[end_line + 1:]
264
264
265 return lines_before + [new_line] + lines_after
265 return lines_before + [new_line] + lines_after
266
266
267 # The escape sequences that define the syntax transformations IPython will
267 # The escape sequences that define the syntax transformations IPython will
268 # apply to user input. These can NOT be just changed here: many regular
268 # apply to user input. These can NOT be just changed here: many regular
269 # expressions and other parts of the code may use their hardcoded values, and
269 # expressions and other parts of the code may use their hardcoded values, and
270 # for all intents and purposes they constitute the 'IPython syntax', so they
270 # for all intents and purposes they constitute the 'IPython syntax', so they
271 # should be considered fixed.
271 # should be considered fixed.
272
272
273 ESC_SHELL = '!' # Send line to underlying system shell
273 ESC_SHELL = '!' # Send line to underlying system shell
274 ESC_SH_CAP = '!!' # Send line to system shell and capture output
274 ESC_SH_CAP = '!!' # Send line to system shell and capture output
275 ESC_HELP = '?' # Find information about object
275 ESC_HELP = '?' # Find information about object
276 ESC_HELP2 = '??' # Find extra-detailed information about object
276 ESC_HELP2 = '??' # Find extra-detailed information about object
277 ESC_MAGIC = '%' # Call magic function
277 ESC_MAGIC = '%' # Call magic function
278 ESC_MAGIC2 = '%%' # Call cell-magic function
278 ESC_MAGIC2 = '%%' # Call cell-magic function
279 ESC_QUOTE = ',' # Split args on whitespace, quote each as string and call
279 ESC_QUOTE = ',' # Split args on whitespace, quote each as string and call
280 ESC_QUOTE2 = ';' # Quote all args as a single string, call
280 ESC_QUOTE2 = ';' # Quote all args as a single string, call
281 ESC_PAREN = '/' # Call first argument with rest of line as arguments
281 ESC_PAREN = '/' # Call first argument with rest of line as arguments
282
282
283 ESCAPE_SINGLES = {'!', '?', '%', ',', ';', '/'}
283 ESCAPE_SINGLES = {'!', '?', '%', ',', ';', '/'}
284 ESCAPE_DOUBLES = {'!!', '??'} # %% (cell magic) is handled separately
284 ESCAPE_DOUBLES = {'!!', '??'} # %% (cell magic) is handled separately
285
285
286 def _make_help_call(target, esc, next_input=None):
286 def _make_help_call(target, esc, next_input=None):
287 """Prepares a pinfo(2)/psearch call from a target name and the escape
287 """Prepares a pinfo(2)/psearch call from a target name and the escape
288 (i.e. ? or ??)"""
288 (i.e. ? or ??)"""
289 method = 'pinfo2' if esc == '??' \
289 method = 'pinfo2' if esc == '??' \
290 else 'psearch' if '*' in target \
290 else 'psearch' if '*' in target \
291 else 'pinfo'
291 else 'pinfo'
292 arg = " ".join([method, target])
292 arg = " ".join([method, target])
293 #Prepare arguments for get_ipython().run_line_magic(magic_name, magic_args)
293 #Prepare arguments for get_ipython().run_line_magic(magic_name, magic_args)
294 t_magic_name, _, t_magic_arg_s = arg.partition(' ')
294 t_magic_name, _, t_magic_arg_s = arg.partition(' ')
295 t_magic_name = t_magic_name.lstrip(ESC_MAGIC)
295 t_magic_name = t_magic_name.lstrip(ESC_MAGIC)
296 if next_input is None:
296 if next_input is None:
297 return 'get_ipython().run_line_magic(%r, %r)' % (t_magic_name, t_magic_arg_s)
297 return 'get_ipython().run_line_magic(%r, %r)' % (t_magic_name, t_magic_arg_s)
298 else:
298 else:
299 return 'get_ipython().set_next_input(%r);get_ipython().run_line_magic(%r, %r)' % \
299 return 'get_ipython().set_next_input(%r);get_ipython().run_line_magic(%r, %r)' % \
300 (next_input, t_magic_name, t_magic_arg_s)
300 (next_input, t_magic_name, t_magic_arg_s)
301
301
302 def _tr_help(content):
302 def _tr_help(content):
303 """Translate lines escaped with: ?
303 """Translate lines escaped with: ?
304
304
305 A naked help line should fire the intro help screen (shell.show_usage())
305 A naked help line should fire the intro help screen (shell.show_usage())
306 """
306 """
307 if not content:
307 if not content:
308 return 'get_ipython().show_usage()'
308 return 'get_ipython().show_usage()'
309
309
310 return _make_help_call(content, '?')
310 return _make_help_call(content, '?')
311
311
312 def _tr_help2(content):
312 def _tr_help2(content):
313 """Translate lines escaped with: ??
313 """Translate lines escaped with: ??
314
314
315 A naked help line should fire the intro help screen (shell.show_usage())
315 A naked help line should fire the intro help screen (shell.show_usage())
316 """
316 """
317 if not content:
317 if not content:
318 return 'get_ipython().show_usage()'
318 return 'get_ipython().show_usage()'
319
319
320 return _make_help_call(content, '??')
320 return _make_help_call(content, '??')
321
321
322 def _tr_magic(content):
322 def _tr_magic(content):
323 "Translate lines escaped with a percent sign: %"
323 "Translate lines escaped with a percent sign: %"
324 name, _, args = content.partition(' ')
324 name, _, args = content.partition(' ')
325 return 'get_ipython().run_line_magic(%r, %r)' % (name, args)
325 return 'get_ipython().run_line_magic(%r, %r)' % (name, args)
326
326
327 def _tr_quote(content):
327 def _tr_quote(content):
328 "Translate lines escaped with a comma: ,"
328 "Translate lines escaped with a comma: ,"
329 name, _, args = content.partition(' ')
329 name, _, args = content.partition(' ')
330 return '%s("%s")' % (name, '", "'.join(args.split()) )
330 return '%s("%s")' % (name, '", "'.join(args.split()) )
331
331
332 def _tr_quote2(content):
332 def _tr_quote2(content):
333 "Translate lines escaped with a semicolon: ;"
333 "Translate lines escaped with a semicolon: ;"
334 name, _, args = content.partition(' ')
334 name, _, args = content.partition(' ')
335 return '%s("%s")' % (name, args)
335 return '%s("%s")' % (name, args)
336
336
337 def _tr_paren(content):
337 def _tr_paren(content):
338 "Translate lines escaped with a slash: /"
338 "Translate lines escaped with a slash: /"
339 name, _, args = content.partition(' ')
339 name, _, args = content.partition(' ')
340 return '%s(%s)' % (name, ", ".join(args.split()))
340 return '%s(%s)' % (name, ", ".join(args.split()))
341
341
342 tr = { ESC_SHELL : 'get_ipython().system({!r})'.format,
342 tr = { ESC_SHELL : 'get_ipython().system({!r})'.format,
343 ESC_SH_CAP : 'get_ipython().getoutput({!r})'.format,
343 ESC_SH_CAP : 'get_ipython().getoutput({!r})'.format,
344 ESC_HELP : _tr_help,
344 ESC_HELP : _tr_help,
345 ESC_HELP2 : _tr_help2,
345 ESC_HELP2 : _tr_help2,
346 ESC_MAGIC : _tr_magic,
346 ESC_MAGIC : _tr_magic,
347 ESC_QUOTE : _tr_quote,
347 ESC_QUOTE : _tr_quote,
348 ESC_QUOTE2 : _tr_quote2,
348 ESC_QUOTE2 : _tr_quote2,
349 ESC_PAREN : _tr_paren }
349 ESC_PAREN : _tr_paren }
350
350
351 class EscapedCommand(TokenTransformBase):
351 class EscapedCommand(TokenTransformBase):
352 """Transformer for escaped commands like %foo, !foo, or /foo"""
352 """Transformer for escaped commands like %foo, !foo, or /foo"""
353 @classmethod
353 @classmethod
354 def find(cls, tokens_by_line):
354 def find(cls, tokens_by_line):
355 """Find the first escaped command (%foo, !foo, etc.) in the cell.
355 """Find the first escaped command (%foo, !foo, etc.) in the cell.
356 """
356 """
357 for line in tokens_by_line:
357 for line in tokens_by_line:
358 if not line:
359 continue
358 ix = 0
360 ix = 0
359 while line[ix].type in {tokenize.INDENT, tokenize.DEDENT}:
361 ll = len(line)
362 while ll > ix and line[ix].type in {tokenize.INDENT, tokenize.DEDENT}:
360 ix += 1
363 ix += 1
364 if ix >= ll:
365 continue
361 if line[ix].string in ESCAPE_SINGLES:
366 if line[ix].string in ESCAPE_SINGLES:
362 return cls(line[ix].start)
367 return cls(line[ix].start)
363
368
364 def transform(self, lines):
369 def transform(self, lines):
365 """Transform an escaped line found by the ``find()`` classmethod.
370 """Transform an escaped line found by the ``find()`` classmethod.
366 """
371 """
367 start_line, start_col = self.start_line, self.start_col
372 start_line, start_col = self.start_line, self.start_col
368
373
369 indent = lines[start_line][:start_col]
374 indent = lines[start_line][:start_col]
370 end_line = find_end_of_continued_line(lines, start_line)
375 end_line = find_end_of_continued_line(lines, start_line)
371 line = assemble_continued_line(lines, (start_line, start_col), end_line)
376 line = assemble_continued_line(lines, (start_line, start_col), end_line)
372
377
373 if len(line) > 1 and line[:2] in ESCAPE_DOUBLES:
378 if len(line) > 1 and line[:2] in ESCAPE_DOUBLES:
374 escape, content = line[:2], line[2:]
379 escape, content = line[:2], line[2:]
375 else:
380 else:
376 escape, content = line[:1], line[1:]
381 escape, content = line[:1], line[1:]
377
382
378 if escape in tr:
383 if escape in tr:
379 call = tr[escape](content)
384 call = tr[escape](content)
380 else:
385 else:
381 call = ''
386 call = ''
382
387
383 lines_before = lines[:start_line]
388 lines_before = lines[:start_line]
384 new_line = indent + call + '\n'
389 new_line = indent + call + '\n'
385 lines_after = lines[end_line + 1:]
390 lines_after = lines[end_line + 1:]
386
391
387 return lines_before + [new_line] + lines_after
392 return lines_before + [new_line] + lines_after
388
393
389 _help_end_re = re.compile(r"""(%{0,2}
394 _help_end_re = re.compile(r"""(%{0,2}
390 [a-zA-Z_*][\w*]* # Variable name
395 [a-zA-Z_*][\w*]* # Variable name
391 (\.[a-zA-Z_*][\w*]*)* # .etc.etc
396 (\.[a-zA-Z_*][\w*]*)* # .etc.etc
392 )
397 )
393 (\?\??)$ # ? or ??
398 (\?\??)$ # ? or ??
394 """,
399 """,
395 re.VERBOSE)
400 re.VERBOSE)
396
401
397 class HelpEnd(TokenTransformBase):
402 class HelpEnd(TokenTransformBase):
398 """Transformer for help syntax: obj? and obj??"""
403 """Transformer for help syntax: obj? and obj??"""
399 # This needs to be higher priority (lower number) than EscapedCommand so
404 # This needs to be higher priority (lower number) than EscapedCommand so
400 # that inspecting magics (%foo?) works.
405 # that inspecting magics (%foo?) works.
401 priority = 5
406 priority = 5
402
407
403 def __init__(self, start, q_locn):
408 def __init__(self, start, q_locn):
404 super().__init__(start)
409 super().__init__(start)
405 self.q_line = q_locn[0] - 1 # Shift from 1-indexed to 0-indexed
410 self.q_line = q_locn[0] - 1 # Shift from 1-indexed to 0-indexed
406 self.q_col = q_locn[1]
411 self.q_col = q_locn[1]
407
412
408 @classmethod
413 @classmethod
409 def find(cls, tokens_by_line):
414 def find(cls, tokens_by_line):
410 """Find the first help command (foo?) in the cell.
415 """Find the first help command (foo?) in the cell.
411 """
416 """
412 for line in tokens_by_line:
417 for line in tokens_by_line:
413 # Last token is NEWLINE; look at last but one
418 # Last token is NEWLINE; look at last but one
414 if len(line) > 2 and line[-2].string == '?':
419 if len(line) > 2 and line[-2].string == '?':
415 # Find the first token that's not INDENT/DEDENT
420 # Find the first token that's not INDENT/DEDENT
416 ix = 0
421 ix = 0
417 while line[ix].type in {tokenize.INDENT, tokenize.DEDENT}:
422 while line[ix].type in {tokenize.INDENT, tokenize.DEDENT}:
418 ix += 1
423 ix += 1
419 return cls(line[ix].start, line[-2].start)
424 return cls(line[ix].start, line[-2].start)
420
425
421 def transform(self, lines):
426 def transform(self, lines):
422 """Transform a help command found by the ``find()`` classmethod.
427 """Transform a help command found by the ``find()`` classmethod.
423 """
428 """
424 piece = ''.join(lines[self.start_line:self.q_line+1])
429 piece = ''.join(lines[self.start_line:self.q_line+1])
425 indent, content = piece[:self.start_col], piece[self.start_col:]
430 indent, content = piece[:self.start_col], piece[self.start_col:]
426 lines_before = lines[:self.start_line]
431 lines_before = lines[:self.start_line]
427 lines_after = lines[self.q_line + 1:]
432 lines_after = lines[self.q_line + 1:]
428
433
429 m = _help_end_re.search(content)
434 m = _help_end_re.search(content)
430 if not m:
435 if not m:
431 raise SyntaxError(content)
436 raise SyntaxError(content)
432 assert m is not None, content
437 assert m is not None, content
433 target = m.group(1)
438 target = m.group(1)
434 esc = m.group(3)
439 esc = m.group(3)
435
440
436 # If we're mid-command, put it back on the next prompt for the user.
441 # If we're mid-command, put it back on the next prompt for the user.
437 next_input = None
442 next_input = None
438 if (not lines_before) and (not lines_after) \
443 if (not lines_before) and (not lines_after) \
439 and content.strip() != m.group(0):
444 and content.strip() != m.group(0):
440 next_input = content.rstrip('?\n')
445 next_input = content.rstrip('?\n')
441
446
442 call = _make_help_call(target, esc, next_input=next_input)
447 call = _make_help_call(target, esc, next_input=next_input)
443 new_line = indent + call + '\n'
448 new_line = indent + call + '\n'
444
449
445 return lines_before + [new_line] + lines_after
450 return lines_before + [new_line] + lines_after
446
451
447 def make_tokens_by_line(lines):
452 def make_tokens_by_line(lines):
448 """Tokenize a series of lines and group tokens by line.
453 """Tokenize a series of lines and group tokens by line.
449
454
450 The tokens for a multiline Python string or expression are
455 The tokens for a multiline Python string or expression are
451 grouped as one line.
456 grouped as one line.
452 """
457 """
453 # NL tokens are used inside multiline expressions, but also after blank
458 # NL tokens are used inside multiline expressions, but also after blank
454 # lines or comments. This is intentional - see https://bugs.python.org/issue17061
459 # lines or comments. This is intentional - see https://bugs.python.org/issue17061
455 # We want to group the former case together but split the latter, so we
460 # We want to group the former case together but split the latter, so we
456 # track parentheses level, similar to the internals of tokenize.
461 # track parentheses level, similar to the internals of tokenize.
457 NEWLINE, NL = tokenize.NEWLINE, tokenize.NL
462 NEWLINE, NL = tokenize.NEWLINE, tokenize.NL
458 tokens_by_line = [[]]
463 tokens_by_line = [[]]
459 parenlev = 0
464 parenlev = 0
460 try:
465 try:
461 for token in tokenize.generate_tokens(iter(lines).__next__):
466 for token in tokenize.generate_tokens(iter(lines).__next__):
462 tokens_by_line[-1].append(token)
467 tokens_by_line[-1].append(token)
463 if (token.type == NEWLINE) \
468 if (token.type == NEWLINE) \
464 or ((token.type == NL) and (parenlev <= 0)):
469 or ((token.type == NL) and (parenlev <= 0)):
465 tokens_by_line.append([])
470 tokens_by_line.append([])
466 elif token.string in {'(', '[', '{'}:
471 elif token.string in {'(', '[', '{'}:
467 parenlev += 1
472 parenlev += 1
468 elif token.string in {')', ']', '}'}:
473 elif token.string in {')', ']', '}'}:
469 if parenlev > 0:
474 if parenlev > 0:
470 parenlev -= 1
475 parenlev -= 1
471 except tokenize.TokenError:
476 except tokenize.TokenError:
472 # Input ended in a multiline string or expression. That's OK for us.
477 # Input ended in a multiline string or expression. That's OK for us.
473 pass
478 pass
474
479
475
480
476 if not tokens_by_line[-1]:
481 if not tokens_by_line[-1]:
477 tokens_by_line.pop()
482 tokens_by_line.pop()
478
483
479
484
480 return tokens_by_line
485 return tokens_by_line
481
486
482 def show_linewise_tokens(s: str):
487 def show_linewise_tokens(s: str):
483 """For investigation and debugging"""
488 """For investigation and debugging"""
484 if not s.endswith('\n'):
489 if not s.endswith('\n'):
485 s += '\n'
490 s += '\n'
486 lines = s.splitlines(keepends=True)
491 lines = s.splitlines(keepends=True)
487 for line in make_tokens_by_line(lines):
492 for line in make_tokens_by_line(lines):
488 print("Line -------")
493 print("Line -------")
489 for tokinfo in line:
494 for tokinfo in line:
490 print(" ", tokinfo)
495 print(" ", tokinfo)
491
496
492 # Arbitrary limit to prevent getting stuck in infinite loops
497 # Arbitrary limit to prevent getting stuck in infinite loops
493 TRANSFORM_LOOP_LIMIT = 500
498 TRANSFORM_LOOP_LIMIT = 500
494
499
495 class TransformerManager:
500 class TransformerManager:
496 """Applies various transformations to a cell or code block.
501 """Applies various transformations to a cell or code block.
497
502
498 The key methods for external use are ``transform_cell()``
503 The key methods for external use are ``transform_cell()``
499 and ``check_complete()``.
504 and ``check_complete()``.
500 """
505 """
501 def __init__(self):
506 def __init__(self):
502 self.cleanup_transforms = [
507 self.cleanup_transforms = [
503 leading_indent,
508 leading_indent,
504 classic_prompt,
509 classic_prompt,
505 ipython_prompt,
510 ipython_prompt,
506 ]
511 ]
507 self.line_transforms = [
512 self.line_transforms = [
508 cell_magic,
513 cell_magic,
509 ]
514 ]
510 self.token_transformers = [
515 self.token_transformers = [
511 MagicAssign,
516 MagicAssign,
512 SystemAssign,
517 SystemAssign,
513 EscapedCommand,
518 EscapedCommand,
514 HelpEnd,
519 HelpEnd,
515 ]
520 ]
516
521
517 def do_one_token_transform(self, lines):
522 def do_one_token_transform(self, lines):
518 """Find and run the transform earliest in the code.
523 """Find and run the transform earliest in the code.
519
524
520 Returns (changed, lines).
525 Returns (changed, lines).
521
526
522 This method is called repeatedly until changed is False, indicating
527 This method is called repeatedly until changed is False, indicating
523 that all available transformations are complete.
528 that all available transformations are complete.
524
529
525 The tokens following IPython special syntax might not be valid, so
530 The tokens following IPython special syntax might not be valid, so
526 the transformed code is retokenised every time to identify the next
531 the transformed code is retokenised every time to identify the next
527 piece of special syntax. Hopefully long code cells are mostly valid
532 piece of special syntax. Hopefully long code cells are mostly valid
528 Python, not using lots of IPython special syntax, so this shouldn't be
533 Python, not using lots of IPython special syntax, so this shouldn't be
529 a performance issue.
534 a performance issue.
530 """
535 """
531 tokens_by_line = make_tokens_by_line(lines)
536 tokens_by_line = make_tokens_by_line(lines)
532 candidates = []
537 candidates = []
533 for transformer_cls in self.token_transformers:
538 for transformer_cls in self.token_transformers:
534 transformer = transformer_cls.find(tokens_by_line)
539 transformer = transformer_cls.find(tokens_by_line)
535 if transformer:
540 if transformer:
536 candidates.append(transformer)
541 candidates.append(transformer)
537
542
538 if not candidates:
543 if not candidates:
539 # Nothing to transform
544 # Nothing to transform
540 return False, lines
545 return False, lines
541 ordered_transformers = sorted(candidates, key=TokenTransformBase.sortby)
546 ordered_transformers = sorted(candidates, key=TokenTransformBase.sortby)
542 for transformer in ordered_transformers:
547 for transformer in ordered_transformers:
543 try:
548 try:
544 return True, transformer.transform(lines)
549 return True, transformer.transform(lines)
545 except SyntaxError:
550 except SyntaxError:
546 pass
551 pass
547 return False, lines
552 return False, lines
548
553
549 def do_token_transforms(self, lines):
554 def do_token_transforms(self, lines):
550 for _ in range(TRANSFORM_LOOP_LIMIT):
555 for _ in range(TRANSFORM_LOOP_LIMIT):
551 changed, lines = self.do_one_token_transform(lines)
556 changed, lines = self.do_one_token_transform(lines)
552 if not changed:
557 if not changed:
553 return lines
558 return lines
554
559
555 raise RuntimeError("Input transformation still changing after "
560 raise RuntimeError("Input transformation still changing after "
556 "%d iterations. Aborting." % TRANSFORM_LOOP_LIMIT)
561 "%d iterations. Aborting." % TRANSFORM_LOOP_LIMIT)
557
562
558 def transform_cell(self, cell: str) -> str:
563 def transform_cell(self, cell: str) -> str:
559 """Transforms a cell of input code"""
564 """Transforms a cell of input code"""
560 if not cell.endswith('\n'):
565 if not cell.endswith('\n'):
561 cell += '\n' # Ensure the cell has a trailing newline
566 cell += '\n' # Ensure the cell has a trailing newline
562 lines = cell.splitlines(keepends=True)
567 lines = cell.splitlines(keepends=True)
563 for transform in self.cleanup_transforms + self.line_transforms:
568 for transform in self.cleanup_transforms + self.line_transforms:
564 lines = transform(lines)
569 lines = transform(lines)
565
570
566 lines = self.do_token_transforms(lines)
571 lines = self.do_token_transforms(lines)
567 return ''.join(lines)
572 return ''.join(lines)
568
573
569 def check_complete(self, cell: str):
574 def check_complete(self, cell: str):
570 """Return whether a block of code is ready to execute, or should be continued
575 """Return whether a block of code is ready to execute, or should be continued
571
576
572 Parameters
577 Parameters
573 ----------
578 ----------
574 source : string
579 source : string
575 Python input code, which can be multiline.
580 Python input code, which can be multiline.
576
581
577 Returns
582 Returns
578 -------
583 -------
579 status : str
584 status : str
580 One of 'complete', 'incomplete', or 'invalid' if source is not a
585 One of 'complete', 'incomplete', or 'invalid' if source is not a
581 prefix of valid code.
586 prefix of valid code.
582 indent_spaces : int or None
587 indent_spaces : int or None
583 The number of spaces by which to indent the next line of code. If
588 The number of spaces by which to indent the next line of code. If
584 status is not 'incomplete', this is None.
589 status is not 'incomplete', this is None.
585 """
590 """
586 # Remember if the lines ends in a new line.
591 # Remember if the lines ends in a new line.
587 ends_with_newline = False
592 ends_with_newline = False
588 for character in reversed(cell):
593 for character in reversed(cell):
589 if character == '\n':
594 if character == '\n':
590 ends_with_newline = True
595 ends_with_newline = True
591 break
596 break
592 elif character.strip():
597 elif character.strip():
593 break
598 break
594 else:
599 else:
595 continue
600 continue
596
601
597 if ends_with_newline:
602 if ends_with_newline:
598 # Append an newline for consistent tokenization
603 # Append an newline for consistent tokenization
599 # See https://bugs.python.org/issue33899
604 # See https://bugs.python.org/issue33899
600 cell += '\n'
605 cell += '\n'
601
606
602 lines = cell.splitlines(keepends=True)
607 lines = cell.splitlines(keepends=True)
603
608
604 if not lines:
609 if not lines:
605 return 'complete', None
610 return 'complete', None
606
611
607 if lines[-1].endswith('\\'):
612 if lines[-1].endswith('\\'):
608 # Explicit backslash continuation
613 # Explicit backslash continuation
609 return 'incomplete', find_last_indent(lines)
614 return 'incomplete', find_last_indent(lines)
610
615
611 try:
616 try:
612 for transform in self.cleanup_transforms:
617 for transform in self.cleanup_transforms:
613 lines = transform(lines)
618 lines = transform(lines)
614 except SyntaxError:
619 except SyntaxError:
615 return 'invalid', None
620 return 'invalid', None
616
621
617 if lines[0].startswith('%%'):
622 if lines[0].startswith('%%'):
618 # Special case for cell magics - completion marked by blank line
623 # Special case for cell magics - completion marked by blank line
619 if lines[-1].strip():
624 if lines[-1].strip():
620 return 'incomplete', find_last_indent(lines)
625 return 'incomplete', find_last_indent(lines)
621 else:
626 else:
622 return 'complete', None
627 return 'complete', None
623
628
624 try:
629 try:
625 for transform in self.line_transforms:
630 for transform in self.line_transforms:
626 lines = transform(lines)
631 lines = transform(lines)
627 lines = self.do_token_transforms(lines)
632 lines = self.do_token_transforms(lines)
628 except SyntaxError:
633 except SyntaxError:
629 return 'invalid', None
634 return 'invalid', None
630
635
631 tokens_by_line = make_tokens_by_line(lines)
636 tokens_by_line = make_tokens_by_line(lines)
632
637
633 if not tokens_by_line:
638 if not tokens_by_line:
634 return 'incomplete', find_last_indent(lines)
639 return 'incomplete', find_last_indent(lines)
635
640
636 if tokens_by_line[-1][-1].type != tokenize.ENDMARKER:
641 if tokens_by_line[-1][-1].type != tokenize.ENDMARKER:
637 # We're in a multiline string or expression
642 # We're in a multiline string or expression
638 return 'incomplete', find_last_indent(lines)
643 return 'incomplete', find_last_indent(lines)
639
644
640 newline_types = {tokenize.NEWLINE, tokenize.COMMENT, tokenize.ENDMARKER}
645 newline_types = {tokenize.NEWLINE, tokenize.COMMENT, tokenize.ENDMARKER}
641
646
642 # Remove newline_types for the list of tokens
647 # Remove newline_types for the list of tokens
643 while len(tokens_by_line) > 1 and len(tokens_by_line[-1]) == 1 \
648 while len(tokens_by_line) > 1 and len(tokens_by_line[-1]) == 1 \
644 and tokens_by_line[-1][-1].type in newline_types:
649 and tokens_by_line[-1][-1].type in newline_types:
645 tokens_by_line.pop()
650 tokens_by_line.pop()
646
651
647 while tokens_by_line[-1] and tokens_by_line[-1][-1].type in newline_types:
652 while tokens_by_line[-1] and tokens_by_line[-1][-1].type in newline_types:
648 tokens_by_line[-1].pop()
653 tokens_by_line[-1].pop()
649
654
650 if len(tokens_by_line) == 1 and not tokens_by_line[-1]:
655 if len(tokens_by_line) == 1 and not tokens_by_line[-1]:
651 return 'incomplete', 0
656 return 'incomplete', 0
652
657
653 if tokens_by_line[-1][-1].string == ':':
658 if tokens_by_line[-1][-1].string == ':':
654 # The last line starts a block (e.g. 'if foo:')
659 # The last line starts a block (e.g. 'if foo:')
655 ix = 0
660 ix = 0
656 while tokens_by_line[-1][ix].type in {tokenize.INDENT, tokenize.DEDENT}:
661 while tokens_by_line[-1][ix].type in {tokenize.INDENT, tokenize.DEDENT}:
657 ix += 1
662 ix += 1
658
663
659 indent = tokens_by_line[-1][ix].start[1]
664 indent = tokens_by_line[-1][ix].start[1]
660 return 'incomplete', indent + 4
665 return 'incomplete', indent + 4
661
666
662 if tokens_by_line[-1][0].line.endswith('\\'):
667 if tokens_by_line[-1][0].line.endswith('\\'):
663 return 'incomplete', None
668 return 'incomplete', None
664
669
665 # At this point, our checks think the code is complete (or invalid).
670 # At this point, our checks think the code is complete (or invalid).
666 # We'll use codeop.compile_command to check this with the real parser
671 # We'll use codeop.compile_command to check this with the real parser
667 try:
672 try:
668 with warnings.catch_warnings():
673 with warnings.catch_warnings():
669 warnings.simplefilter('error', SyntaxWarning)
674 warnings.simplefilter('error', SyntaxWarning)
670 res = compile_command(''.join(lines), symbol='exec')
675 res = compile_command(''.join(lines), symbol='exec')
671 except (SyntaxError, OverflowError, ValueError, TypeError,
676 except (SyntaxError, OverflowError, ValueError, TypeError,
672 MemoryError, SyntaxWarning):
677 MemoryError, SyntaxWarning):
673 return 'invalid', None
678 return 'invalid', None
674 else:
679 else:
675 if res is None:
680 if res is None:
676 return 'incomplete', find_last_indent(lines)
681 return 'incomplete', find_last_indent(lines)
677
682
678 if tokens_by_line[-1][-1].type == tokenize.DEDENT:
683 if tokens_by_line[-1][-1].type == tokenize.DEDENT:
679 if ends_with_newline:
684 if ends_with_newline:
680 return 'complete', None
685 return 'complete', None
681 return 'incomplete', find_last_indent(lines)
686 return 'incomplete', find_last_indent(lines)
682
687
683 # If there's a blank line at the end, assume we're ready to execute
688 # If there's a blank line at the end, assume we're ready to execute
684 if not lines[-1].strip():
689 if not lines[-1].strip():
685 return 'complete', None
690 return 'complete', None
686
691
687 return 'complete', None
692 return 'complete', None
688
693
689
694
690 def find_last_indent(lines):
695 def find_last_indent(lines):
691 m = _indent_re.match(lines[-1])
696 m = _indent_re.match(lines[-1])
692 if not m:
697 if not m:
693 return 0
698 return 0
694 return len(m.group(0).replace('\t', ' '*4))
699 return len(m.group(0).replace('\t', ' '*4))
@@ -1,239 +1,250 b''
1 """Tests for the token-based transformers in IPython.core.inputtransformer2
1 """Tests for the token-based transformers in IPython.core.inputtransformer2
2
2
3 Line-based transformers are the simpler ones; token-based transformers are
3 Line-based transformers are the simpler ones; token-based transformers are
4 more complex. See test_inputtransformer2_line for tests for line-based
4 more complex. See test_inputtransformer2_line for tests for line-based
5 transformations.
5 transformations.
6 """
6 """
7 import nose.tools as nt
7 import nose.tools as nt
8 import string
8 import string
9
9
10 from IPython.core import inputtransformer2 as ipt2
10 from IPython.core import inputtransformer2 as ipt2
11 from IPython.core.inputtransformer2 import make_tokens_by_line
11 from IPython.core.inputtransformer2 import make_tokens_by_line
12
12
13 from textwrap import dedent
13 from textwrap import dedent
14
14
15 MULTILINE_MAGIC = ("""\
15 MULTILINE_MAGIC = ("""\
16 a = f()
16 a = f()
17 %foo \\
17 %foo \\
18 bar
18 bar
19 g()
19 g()
20 """.splitlines(keepends=True), (2, 0), """\
20 """.splitlines(keepends=True), (2, 0), """\
21 a = f()
21 a = f()
22 get_ipython().run_line_magic('foo', ' bar')
22 get_ipython().run_line_magic('foo', ' bar')
23 g()
23 g()
24 """.splitlines(keepends=True))
24 """.splitlines(keepends=True))
25
25
26 INDENTED_MAGIC = ("""\
26 INDENTED_MAGIC = ("""\
27 for a in range(5):
27 for a in range(5):
28 %ls
28 %ls
29 """.splitlines(keepends=True), (2, 4), """\
29 """.splitlines(keepends=True), (2, 4), """\
30 for a in range(5):
30 for a in range(5):
31 get_ipython().run_line_magic('ls', '')
31 get_ipython().run_line_magic('ls', '')
32 """.splitlines(keepends=True))
32 """.splitlines(keepends=True))
33
33
34 MULTILINE_MAGIC_ASSIGN = ("""\
34 MULTILINE_MAGIC_ASSIGN = ("""\
35 a = f()
35 a = f()
36 b = %foo \\
36 b = %foo \\
37 bar
37 bar
38 g()
38 g()
39 """.splitlines(keepends=True), (2, 4), """\
39 """.splitlines(keepends=True), (2, 4), """\
40 a = f()
40 a = f()
41 b = get_ipython().run_line_magic('foo', ' bar')
41 b = get_ipython().run_line_magic('foo', ' bar')
42 g()
42 g()
43 """.splitlines(keepends=True))
43 """.splitlines(keepends=True))
44
44
45 MULTILINE_SYSTEM_ASSIGN = ("""\
45 MULTILINE_SYSTEM_ASSIGN = ("""\
46 a = f()
46 a = f()
47 b = !foo \\
47 b = !foo \\
48 bar
48 bar
49 g()
49 g()
50 """.splitlines(keepends=True), (2, 4), """\
50 """.splitlines(keepends=True), (2, 4), """\
51 a = f()
51 a = f()
52 b = get_ipython().getoutput('foo bar')
52 b = get_ipython().getoutput('foo bar')
53 g()
53 g()
54 """.splitlines(keepends=True))
54 """.splitlines(keepends=True))
55
55
56 AUTOCALL_QUOTE = (
56 AUTOCALL_QUOTE = (
57 [",f 1 2 3\n"], (1, 0),
57 [",f 1 2 3\n"], (1, 0),
58 ['f("1", "2", "3")\n']
58 ['f("1", "2", "3")\n']
59 )
59 )
60
60
61 AUTOCALL_QUOTE2 = (
61 AUTOCALL_QUOTE2 = (
62 [";f 1 2 3\n"], (1, 0),
62 [";f 1 2 3\n"], (1, 0),
63 ['f("1 2 3")\n']
63 ['f("1 2 3")\n']
64 )
64 )
65
65
66 AUTOCALL_PAREN = (
66 AUTOCALL_PAREN = (
67 ["/f 1 2 3\n"], (1, 0),
67 ["/f 1 2 3\n"], (1, 0),
68 ['f(1, 2, 3)\n']
68 ['f(1, 2, 3)\n']
69 )
69 )
70
70
71 SIMPLE_HELP = (
71 SIMPLE_HELP = (
72 ["foo?\n"], (1, 0),
72 ["foo?\n"], (1, 0),
73 ["get_ipython().run_line_magic('pinfo', 'foo')\n"]
73 ["get_ipython().run_line_magic('pinfo', 'foo')\n"]
74 )
74 )
75
75
76 DETAILED_HELP = (
76 DETAILED_HELP = (
77 ["foo??\n"], (1, 0),
77 ["foo??\n"], (1, 0),
78 ["get_ipython().run_line_magic('pinfo2', 'foo')\n"]
78 ["get_ipython().run_line_magic('pinfo2', 'foo')\n"]
79 )
79 )
80
80
81 MAGIC_HELP = (
81 MAGIC_HELP = (
82 ["%foo?\n"], (1, 0),
82 ["%foo?\n"], (1, 0),
83 ["get_ipython().run_line_magic('pinfo', '%foo')\n"]
83 ["get_ipython().run_line_magic('pinfo', '%foo')\n"]
84 )
84 )
85
85
86 HELP_IN_EXPR = (
86 HELP_IN_EXPR = (
87 ["a = b + c?\n"], (1, 0),
87 ["a = b + c?\n"], (1, 0),
88 ["get_ipython().set_next_input('a = b + c');"
88 ["get_ipython().set_next_input('a = b + c');"
89 "get_ipython().run_line_magic('pinfo', 'c')\n"]
89 "get_ipython().run_line_magic('pinfo', 'c')\n"]
90 )
90 )
91
91
92 HELP_CONTINUED_LINE = ("""\
92 HELP_CONTINUED_LINE = ("""\
93 a = \\
93 a = \\
94 zip?
94 zip?
95 """.splitlines(keepends=True), (1, 0),
95 """.splitlines(keepends=True), (1, 0),
96 [r"get_ipython().set_next_input('a = \\\nzip');get_ipython().run_line_magic('pinfo', 'zip')" + "\n"]
96 [r"get_ipython().set_next_input('a = \\\nzip');get_ipython().run_line_magic('pinfo', 'zip')" + "\n"]
97 )
97 )
98
98
99 HELP_MULTILINE = ("""\
99 HELP_MULTILINE = ("""\
100 (a,
100 (a,
101 b) = zip?
101 b) = zip?
102 """.splitlines(keepends=True), (1, 0),
102 """.splitlines(keepends=True), (1, 0),
103 [r"get_ipython().set_next_input('(a,\nb) = zip');get_ipython().run_line_magic('pinfo', 'zip')" + "\n"]
103 [r"get_ipython().set_next_input('(a,\nb) = zip');get_ipython().run_line_magic('pinfo', 'zip')" + "\n"]
104 )
104 )
105
105
106 def null_cleanup_transformer(lines):
106 def null_cleanup_transformer(lines):
107 """
107 """
108 A cleanup transform that returns an empty list.
108 A cleanup transform that returns an empty list.
109 """
109 """
110 return []
110 return []
111
111
112 def check_make_token_by_line_never_ends_empty():
112 def check_make_token_by_line_never_ends_empty():
113 """
113 """
114 Check that not sequence of single or double characters ends up leading to en empty list of tokens
114 Check that not sequence of single or double characters ends up leading to en empty list of tokens
115 """
115 """
116 from string import printable
116 from string import printable
117 for c in printable:
117 for c in printable:
118 nt.assert_not_equal(make_tokens_by_line(c)[-1], [])
118 nt.assert_not_equal(make_tokens_by_line(c)[-1], [])
119 for k in printable:
119 for k in printable:
120 nt.assert_not_equal(make_tokens_by_line(c+k)[-1], [])
120 nt.assert_not_equal(make_tokens_by_line(c+k)[-1], [])
121
121
122 def check_find(transformer, case, match=True):
122 def check_find(transformer, case, match=True):
123 sample, expected_start, _ = case
123 sample, expected_start, _ = case
124 tbl = make_tokens_by_line(sample)
124 tbl = make_tokens_by_line(sample)
125 res = transformer.find(tbl)
125 res = transformer.find(tbl)
126 if match:
126 if match:
127 # start_line is stored 0-indexed, expected values are 1-indexed
127 # start_line is stored 0-indexed, expected values are 1-indexed
128 nt.assert_equal((res.start_line+1, res.start_col), expected_start)
128 nt.assert_equal((res.start_line+1, res.start_col), expected_start)
129 return res
129 return res
130 else:
130 else:
131 nt.assert_is(res, None)
131 nt.assert_is(res, None)
132
132
133 def check_transform(transformer_cls, case):
133 def check_transform(transformer_cls, case):
134 lines, start, expected = case
134 lines, start, expected = case
135 transformer = transformer_cls(start)
135 transformer = transformer_cls(start)
136 nt.assert_equal(transformer.transform(lines), expected)
136 nt.assert_equal(transformer.transform(lines), expected)
137
137
138 def test_continued_line():
138 def test_continued_line():
139 lines = MULTILINE_MAGIC_ASSIGN[0]
139 lines = MULTILINE_MAGIC_ASSIGN[0]
140 nt.assert_equal(ipt2.find_end_of_continued_line(lines, 1), 2)
140 nt.assert_equal(ipt2.find_end_of_continued_line(lines, 1), 2)
141
141
142 nt.assert_equal(ipt2.assemble_continued_line(lines, (1, 5), 2), "foo bar")
142 nt.assert_equal(ipt2.assemble_continued_line(lines, (1, 5), 2), "foo bar")
143
143
144 def test_find_assign_magic():
144 def test_find_assign_magic():
145 check_find(ipt2.MagicAssign, MULTILINE_MAGIC_ASSIGN)
145 check_find(ipt2.MagicAssign, MULTILINE_MAGIC_ASSIGN)
146 check_find(ipt2.MagicAssign, MULTILINE_SYSTEM_ASSIGN, match=False)
146 check_find(ipt2.MagicAssign, MULTILINE_SYSTEM_ASSIGN, match=False)
147
147
148 def test_transform_assign_magic():
148 def test_transform_assign_magic():
149 check_transform(ipt2.MagicAssign, MULTILINE_MAGIC_ASSIGN)
149 check_transform(ipt2.MagicAssign, MULTILINE_MAGIC_ASSIGN)
150
150
151 def test_find_assign_system():
151 def test_find_assign_system():
152 check_find(ipt2.SystemAssign, MULTILINE_SYSTEM_ASSIGN)
152 check_find(ipt2.SystemAssign, MULTILINE_SYSTEM_ASSIGN)
153 check_find(ipt2.SystemAssign, (["a = !ls\n"], (1, 5), None))
153 check_find(ipt2.SystemAssign, (["a = !ls\n"], (1, 5), None))
154 check_find(ipt2.SystemAssign, (["a=!ls\n"], (1, 2), None))
154 check_find(ipt2.SystemAssign, (["a=!ls\n"], (1, 2), None))
155 check_find(ipt2.SystemAssign, MULTILINE_MAGIC_ASSIGN, match=False)
155 check_find(ipt2.SystemAssign, MULTILINE_MAGIC_ASSIGN, match=False)
156
156
157 def test_transform_assign_system():
157 def test_transform_assign_system():
158 check_transform(ipt2.SystemAssign, MULTILINE_SYSTEM_ASSIGN)
158 check_transform(ipt2.SystemAssign, MULTILINE_SYSTEM_ASSIGN)
159
159
160 def test_find_magic_escape():
160 def test_find_magic_escape():
161 check_find(ipt2.EscapedCommand, MULTILINE_MAGIC)
161 check_find(ipt2.EscapedCommand, MULTILINE_MAGIC)
162 check_find(ipt2.EscapedCommand, INDENTED_MAGIC)
162 check_find(ipt2.EscapedCommand, INDENTED_MAGIC)
163 check_find(ipt2.EscapedCommand, MULTILINE_MAGIC_ASSIGN, match=False)
163 check_find(ipt2.EscapedCommand, MULTILINE_MAGIC_ASSIGN, match=False)
164
164
165 def test_transform_magic_escape():
165 def test_transform_magic_escape():
166 check_transform(ipt2.EscapedCommand, MULTILINE_MAGIC)
166 check_transform(ipt2.EscapedCommand, MULTILINE_MAGIC)
167 check_transform(ipt2.EscapedCommand, INDENTED_MAGIC)
167 check_transform(ipt2.EscapedCommand, INDENTED_MAGIC)
168
168
169 def test_find_autocalls():
169 def test_find_autocalls():
170 for case in [AUTOCALL_QUOTE, AUTOCALL_QUOTE2, AUTOCALL_PAREN]:
170 for case in [AUTOCALL_QUOTE, AUTOCALL_QUOTE2, AUTOCALL_PAREN]:
171 print("Testing %r" % case[0])
171 print("Testing %r" % case[0])
172 check_find(ipt2.EscapedCommand, case)
172 check_find(ipt2.EscapedCommand, case)
173
173
174 def test_transform_autocall():
174 def test_transform_autocall():
175 for case in [AUTOCALL_QUOTE, AUTOCALL_QUOTE2, AUTOCALL_PAREN]:
175 for case in [AUTOCALL_QUOTE, AUTOCALL_QUOTE2, AUTOCALL_PAREN]:
176 print("Testing %r" % case[0])
176 print("Testing %r" % case[0])
177 check_transform(ipt2.EscapedCommand, case)
177 check_transform(ipt2.EscapedCommand, case)
178
178
179 def test_find_help():
179 def test_find_help():
180 for case in [SIMPLE_HELP, DETAILED_HELP, MAGIC_HELP, HELP_IN_EXPR]:
180 for case in [SIMPLE_HELP, DETAILED_HELP, MAGIC_HELP, HELP_IN_EXPR]:
181 check_find(ipt2.HelpEnd, case)
181 check_find(ipt2.HelpEnd, case)
182
182
183 tf = check_find(ipt2.HelpEnd, HELP_CONTINUED_LINE)
183 tf = check_find(ipt2.HelpEnd, HELP_CONTINUED_LINE)
184 nt.assert_equal(tf.q_line, 1)
184 nt.assert_equal(tf.q_line, 1)
185 nt.assert_equal(tf.q_col, 3)
185 nt.assert_equal(tf.q_col, 3)
186
186
187 tf = check_find(ipt2.HelpEnd, HELP_MULTILINE)
187 tf = check_find(ipt2.HelpEnd, HELP_MULTILINE)
188 nt.assert_equal(tf.q_line, 1)
188 nt.assert_equal(tf.q_line, 1)
189 nt.assert_equal(tf.q_col, 8)
189 nt.assert_equal(tf.q_col, 8)
190
190
191 # ? in a comment does not trigger help
191 # ? in a comment does not trigger help
192 check_find(ipt2.HelpEnd, (["foo # bar?\n"], None, None), match=False)
192 check_find(ipt2.HelpEnd, (["foo # bar?\n"], None, None), match=False)
193 # Nor in a string
193 # Nor in a string
194 check_find(ipt2.HelpEnd, (["foo = '''bar?\n"], None, None), match=False)
194 check_find(ipt2.HelpEnd, (["foo = '''bar?\n"], None, None), match=False)
195
195
196 def test_transform_help():
196 def test_transform_help():
197 tf = ipt2.HelpEnd((1, 0), (1, 9))
197 tf = ipt2.HelpEnd((1, 0), (1, 9))
198 nt.assert_equal(tf.transform(HELP_IN_EXPR[0]), HELP_IN_EXPR[2])
198 nt.assert_equal(tf.transform(HELP_IN_EXPR[0]), HELP_IN_EXPR[2])
199
199
200 tf = ipt2.HelpEnd((1, 0), (2, 3))
200 tf = ipt2.HelpEnd((1, 0), (2, 3))
201 nt.assert_equal(tf.transform(HELP_CONTINUED_LINE[0]), HELP_CONTINUED_LINE[2])
201 nt.assert_equal(tf.transform(HELP_CONTINUED_LINE[0]), HELP_CONTINUED_LINE[2])
202
202
203 tf = ipt2.HelpEnd((1, 0), (2, 8))
203 tf = ipt2.HelpEnd((1, 0), (2, 8))
204 nt.assert_equal(tf.transform(HELP_MULTILINE[0]), HELP_MULTILINE[2])
204 nt.assert_equal(tf.transform(HELP_MULTILINE[0]), HELP_MULTILINE[2])
205
205
206 def test_check_complete():
206 def test_check_complete():
207 cc = ipt2.TransformerManager().check_complete
207 cc = ipt2.TransformerManager().check_complete
208 nt.assert_equal(cc("a = 1"), ('complete', None))
208 nt.assert_equal(cc("a = 1"), ('complete', None))
209 nt.assert_equal(cc("for a in range(5):"), ('incomplete', 4))
209 nt.assert_equal(cc("for a in range(5):"), ('incomplete', 4))
210 nt.assert_equal(cc("raise = 2"), ('invalid', None))
210 nt.assert_equal(cc("raise = 2"), ('invalid', None))
211 nt.assert_equal(cc("a = [1,\n2,"), ('incomplete', 0))
211 nt.assert_equal(cc("a = [1,\n2,"), ('incomplete', 0))
212 nt.assert_equal(cc(")"), ('incomplete', 0))
212 nt.assert_equal(cc(")"), ('incomplete', 0))
213 nt.assert_equal(cc("\\\r\n"), ('incomplete', 0))
213 nt.assert_equal(cc("\\\r\n"), ('incomplete', 0))
214 nt.assert_equal(cc("a = '''\n hi"), ('incomplete', 3))
214 nt.assert_equal(cc("a = '''\n hi"), ('incomplete', 3))
215 nt.assert_equal(cc("def a():\n x=1\n global x"), ('invalid', None))
215 nt.assert_equal(cc("def a():\n x=1\n global x"), ('invalid', None))
216 nt.assert_equal(cc("a \\ "), ('invalid', None)) # Nothing allowed after backslash
216 nt.assert_equal(cc("a \\ "), ('invalid', None)) # Nothing allowed after backslash
217 nt.assert_equal(cc("1\\\n+2"), ('complete', None))
217 nt.assert_equal(cc("1\\\n+2"), ('complete', None))
218 nt.assert_equal(cc("exit"), ('complete', None))
218 nt.assert_equal(cc("exit"), ('complete', None))
219
219
220 example = dedent("""
220 example = dedent("""
221 if True:
221 if True:
222 a=1""" )
222 a=1""" )
223
223
224 nt.assert_equal(cc(example), ('incomplete', 4))
224 nt.assert_equal(cc(example), ('incomplete', 4))
225 nt.assert_equal(cc(example+'\n'), ('complete', None))
225 nt.assert_equal(cc(example+'\n'), ('complete', None))
226 nt.assert_equal(cc(example+'\n '), ('complete', None))
226 nt.assert_equal(cc(example+'\n '), ('complete', None))
227
227
228 # no need to loop on all the letters/numbers.
228 # no need to loop on all the letters/numbers.
229 short = '12abAB'+string.printable[62:]
229 short = '12abAB'+string.printable[62:]
230 for c in short:
230 for c in short:
231 # test does not raise:
231 # test does not raise:
232 cc(c)
232 cc(c)
233 for k in short:
233 for k in short:
234 cc(c+k)
234 cc(c+k)
235
235
236 def test_check_complete_II():
237 """
238 Test that multiple line strings are properly handled.
239
240 Separate test function for convenience
241
242 """
243 cc = ipt2.TransformerManager().check_complete
244 nt.assert_equal(cc('''def foo():\n """'''), ('incomplete', 4))
245
246
236 def test_null_cleanup_transformer():
247 def test_null_cleanup_transformer():
237 manager = ipt2.TransformerManager()
248 manager = ipt2.TransformerManager()
238 manager.cleanup_transforms.insert(0, null_cleanup_transformer)
249 manager.cleanup_transforms.insert(0, null_cleanup_transformer)
239 nt.assert_is(manager.transform_cell(""), "")
250 nt.assert_is(manager.transform_cell(""), "")
General Comments 0
You need to be logged in to leave comments. Login now