##// END OF EJS Templates
Remove a condition from check_complete...
Tony Fast -
Show More
@@ -1,698 +1,694
1 """Input transformer machinery to support IPython special syntax.
1 """Input transformer machinery to support IPython special syntax.
2
2
3 This includes the machinery to recognise and transform ``%magic`` commands,
3 This includes the machinery to recognise and transform ``%magic`` commands,
4 ``!system`` commands, ``help?`` querying, prompt stripping, and so forth.
4 ``!system`` commands, ``help?`` querying, prompt stripping, and so forth.
5
5
6 Added: IPython 7.0. Replaces inputsplitter and inputtransformer which were
6 Added: IPython 7.0. Replaces inputsplitter and inputtransformer which were
7 deprecated in 7.0.
7 deprecated in 7.0.
8 """
8 """
9
9
10 # Copyright (c) IPython Development Team.
10 # Copyright (c) IPython Development Team.
11 # Distributed under the terms of the Modified BSD License.
11 # Distributed under the terms of the Modified BSD License.
12
12
13 from codeop import compile_command
13 from codeop import compile_command
14 import re
14 import re
15 import tokenize
15 import tokenize
16 from typing import List, Tuple
16 from typing import List, Tuple
17 import warnings
17 import warnings
18
18
19 _indent_re = re.compile(r'^[ \t]+')
19 _indent_re = re.compile(r'^[ \t]+')
20
20
21 def leading_indent(lines):
21 def leading_indent(lines):
22 """Remove leading indentation.
22 """Remove leading indentation.
23
23
24 If the first line starts with a spaces or tabs, the same whitespace will be
24 If the first line starts with a spaces or tabs, the same whitespace will be
25 removed from each following line in the cell.
25 removed from each following line in the cell.
26 """
26 """
27 if not lines:
27 if not lines:
28 return lines
28 return lines
29 m = _indent_re.match(lines[0])
29 m = _indent_re.match(lines[0])
30 if not m:
30 if not m:
31 return lines
31 return lines
32 space = m.group(0)
32 space = m.group(0)
33 n = len(space)
33 n = len(space)
34 return [l[n:] if l.startswith(space) else l
34 return [l[n:] if l.startswith(space) else l
35 for l in lines]
35 for l in lines]
36
36
37 class PromptStripper:
37 class PromptStripper:
38 """Remove matching input prompts from a block of input.
38 """Remove matching input prompts from a block of input.
39
39
40 Parameters
40 Parameters
41 ----------
41 ----------
42 prompt_re : regular expression
42 prompt_re : regular expression
43 A regular expression matching any input prompt (including continuation,
43 A regular expression matching any input prompt (including continuation,
44 e.g. ``...``)
44 e.g. ``...``)
45 initial_re : regular expression, optional
45 initial_re : regular expression, optional
46 A regular expression matching only the initial prompt, but not continuation.
46 A regular expression matching only the initial prompt, but not continuation.
47 If no initial expression is given, prompt_re will be used everywhere.
47 If no initial expression is given, prompt_re will be used everywhere.
48 Used mainly for plain Python prompts (``>>>``), where the continuation prompt
48 Used mainly for plain Python prompts (``>>>``), where the continuation prompt
49 ``...`` is a valid Python expression in Python 3, so shouldn't be stripped.
49 ``...`` is a valid Python expression in Python 3, so shouldn't be stripped.
50
50
51 If initial_re and prompt_re differ,
51 If initial_re and prompt_re differ,
52 only initial_re will be tested against the first line.
52 only initial_re will be tested against the first line.
53 If any prompt is found on the first two lines,
53 If any prompt is found on the first two lines,
54 prompts will be stripped from the rest of the block.
54 prompts will be stripped from the rest of the block.
55 """
55 """
56 def __init__(self, prompt_re, initial_re=None):
56 def __init__(self, prompt_re, initial_re=None):
57 self.prompt_re = prompt_re
57 self.prompt_re = prompt_re
58 self.initial_re = initial_re or prompt_re
58 self.initial_re = initial_re or prompt_re
59
59
60 def _strip(self, lines):
60 def _strip(self, lines):
61 return [self.prompt_re.sub('', l, count=1) for l in lines]
61 return [self.prompt_re.sub('', l, count=1) for l in lines]
62
62
63 def __call__(self, lines):
63 def __call__(self, lines):
64 if not lines:
64 if not lines:
65 return lines
65 return lines
66 if self.initial_re.match(lines[0]) or \
66 if self.initial_re.match(lines[0]) or \
67 (len(lines) > 1 and self.prompt_re.match(lines[1])):
67 (len(lines) > 1 and self.prompt_re.match(lines[1])):
68 return self._strip(lines)
68 return self._strip(lines)
69 return lines
69 return lines
70
70
71 classic_prompt = PromptStripper(
71 classic_prompt = PromptStripper(
72 prompt_re=re.compile(r'^(>>>|\.\.\.)( |$)'),
72 prompt_re=re.compile(r'^(>>>|\.\.\.)( |$)'),
73 initial_re=re.compile(r'^>>>( |$)')
73 initial_re=re.compile(r'^>>>( |$)')
74 )
74 )
75
75
76 ipython_prompt = PromptStripper(re.compile(r'^(In \[\d+\]: |\s*\.{3,}: ?)'))
76 ipython_prompt = PromptStripper(re.compile(r'^(In \[\d+\]: |\s*\.{3,}: ?)'))
77
77
78 def cell_magic(lines):
78 def cell_magic(lines):
79 if not lines or not lines[0].startswith('%%'):
79 if not lines or not lines[0].startswith('%%'):
80 return lines
80 return lines
81 if re.match('%%\w+\?', lines[0]):
81 if re.match('%%\w+\?', lines[0]):
82 # This case will be handled by help_end
82 # This case will be handled by help_end
83 return lines
83 return lines
84 magic_name, _, first_line = lines[0][2:-1].partition(' ')
84 magic_name, _, first_line = lines[0][2:-1].partition(' ')
85 body = ''.join(lines[1:])
85 body = ''.join(lines[1:])
86 return ['get_ipython().run_cell_magic(%r, %r, %r)\n'
86 return ['get_ipython().run_cell_magic(%r, %r, %r)\n'
87 % (magic_name, first_line, body)]
87 % (magic_name, first_line, body)]
88
88
89
89
90 def _find_assign_op(token_line):
90 def _find_assign_op(token_line):
91 """Get the index of the first assignment in the line ('=' not inside brackets)
91 """Get the index of the first assignment in the line ('=' not inside brackets)
92
92
93 Note: We don't try to support multiple special assignment (a = b = %foo)
93 Note: We don't try to support multiple special assignment (a = b = %foo)
94 """
94 """
95 paren_level = 0
95 paren_level = 0
96 for i, ti in enumerate(token_line):
96 for i, ti in enumerate(token_line):
97 s = ti.string
97 s = ti.string
98 if s == '=' and paren_level == 0:
98 if s == '=' and paren_level == 0:
99 return i
99 return i
100 if s in '([{':
100 if s in '([{':
101 paren_level += 1
101 paren_level += 1
102 elif s in ')]}':
102 elif s in ')]}':
103 if paren_level > 0:
103 if paren_level > 0:
104 paren_level -= 1
104 paren_level -= 1
105
105
106 def find_end_of_continued_line(lines, start_line: int):
106 def find_end_of_continued_line(lines, start_line: int):
107 """Find the last line of a line explicitly extended using backslashes.
107 """Find the last line of a line explicitly extended using backslashes.
108
108
109 Uses 0-indexed line numbers.
109 Uses 0-indexed line numbers.
110 """
110 """
111 end_line = start_line
111 end_line = start_line
112 while lines[end_line].endswith('\\\n'):
112 while lines[end_line].endswith('\\\n'):
113 end_line += 1
113 end_line += 1
114 if end_line >= len(lines):
114 if end_line >= len(lines):
115 break
115 break
116 return end_line
116 return end_line
117
117
118 def assemble_continued_line(lines, start: Tuple[int, int], end_line: int):
118 def assemble_continued_line(lines, start: Tuple[int, int], end_line: int):
119 """Assemble a single line from multiple continued line pieces
119 """Assemble a single line from multiple continued line pieces
120
120
121 Continued lines are lines ending in ``\``, and the line following the last
121 Continued lines are lines ending in ``\``, and the line following the last
122 ``\`` in the block.
122 ``\`` in the block.
123
123
124 For example, this code continues over multiple lines::
124 For example, this code continues over multiple lines::
125
125
126 if (assign_ix is not None) \
126 if (assign_ix is not None) \
127 and (len(line) >= assign_ix + 2) \
127 and (len(line) >= assign_ix + 2) \
128 and (line[assign_ix+1].string == '%') \
128 and (line[assign_ix+1].string == '%') \
129 and (line[assign_ix+2].type == tokenize.NAME):
129 and (line[assign_ix+2].type == tokenize.NAME):
130
130
131 This statement contains four continued line pieces.
131 This statement contains four continued line pieces.
132 Assembling these pieces into a single line would give::
132 Assembling these pieces into a single line would give::
133
133
134 if (assign_ix is not None) and (len(line) >= assign_ix + 2) and (line[...
134 if (assign_ix is not None) and (len(line) >= assign_ix + 2) and (line[...
135
135
136 This uses 0-indexed line numbers. *start* is (lineno, colno).
136 This uses 0-indexed line numbers. *start* is (lineno, colno).
137
137
138 Used to allow ``%magic`` and ``!system`` commands to be continued over
138 Used to allow ``%magic`` and ``!system`` commands to be continued over
139 multiple lines.
139 multiple lines.
140 """
140 """
141 parts = [lines[start[0]][start[1]:]] + lines[start[0]+1:end_line+1]
141 parts = [lines[start[0]][start[1]:]] + lines[start[0]+1:end_line+1]
142 return ' '.join([p[:-2] for p in parts[:-1]] # Strip backslash+newline
142 return ' '.join([p[:-2] for p in parts[:-1]] # Strip backslash+newline
143 + [parts[-1][:-1]]) # Strip newline from last line
143 + [parts[-1][:-1]]) # Strip newline from last line
144
144
145 class TokenTransformBase:
145 class TokenTransformBase:
146 """Base class for transformations which examine tokens.
146 """Base class for transformations which examine tokens.
147
147
148 Special syntax should not be transformed when it occurs inside strings or
148 Special syntax should not be transformed when it occurs inside strings or
149 comments. This is hard to reliably avoid with regexes. The solution is to
149 comments. This is hard to reliably avoid with regexes. The solution is to
150 tokenise the code as Python, and recognise the special syntax in the tokens.
150 tokenise the code as Python, and recognise the special syntax in the tokens.
151
151
152 IPython's special syntax is not valid Python syntax, so tokenising may go
152 IPython's special syntax is not valid Python syntax, so tokenising may go
153 wrong after the special syntax starts. These classes therefore find and
153 wrong after the special syntax starts. These classes therefore find and
154 transform *one* instance of special syntax at a time into regular Python
154 transform *one* instance of special syntax at a time into regular Python
155 syntax. After each transformation, tokens are regenerated to find the next
155 syntax. After each transformation, tokens are regenerated to find the next
156 piece of special syntax.
156 piece of special syntax.
157
157
158 Subclasses need to implement one class method (find)
158 Subclasses need to implement one class method (find)
159 and one regular method (transform).
159 and one regular method (transform).
160
160
161 The priority attribute can select which transformation to apply if multiple
161 The priority attribute can select which transformation to apply if multiple
162 transformers match in the same place. Lower numbers have higher priority.
162 transformers match in the same place. Lower numbers have higher priority.
163 This allows "%magic?" to be turned into a help call rather than a magic call.
163 This allows "%magic?" to be turned into a help call rather than a magic call.
164 """
164 """
165 # Lower numbers -> higher priority (for matches in the same location)
165 # Lower numbers -> higher priority (for matches in the same location)
166 priority = 10
166 priority = 10
167
167
168 def sortby(self):
168 def sortby(self):
169 return self.start_line, self.start_col, self.priority
169 return self.start_line, self.start_col, self.priority
170
170
171 def __init__(self, start):
171 def __init__(self, start):
172 self.start_line = start[0] - 1 # Shift from 1-index to 0-index
172 self.start_line = start[0] - 1 # Shift from 1-index to 0-index
173 self.start_col = start[1]
173 self.start_col = start[1]
174
174
175 @classmethod
175 @classmethod
176 def find(cls, tokens_by_line):
176 def find(cls, tokens_by_line):
177 """Find one instance of special syntax in the provided tokens.
177 """Find one instance of special syntax in the provided tokens.
178
178
179 Tokens are grouped into logical lines for convenience,
179 Tokens are grouped into logical lines for convenience,
180 so it is easy to e.g. look at the first token of each line.
180 so it is easy to e.g. look at the first token of each line.
181 *tokens_by_line* is a list of lists of tokenize.TokenInfo objects.
181 *tokens_by_line* is a list of lists of tokenize.TokenInfo objects.
182
182
183 This should return an instance of its class, pointing to the start
183 This should return an instance of its class, pointing to the start
184 position it has found, or None if it found no match.
184 position it has found, or None if it found no match.
185 """
185 """
186 raise NotImplementedError
186 raise NotImplementedError
187
187
188 def transform(self, lines: List[str]):
188 def transform(self, lines: List[str]):
189 """Transform one instance of special syntax found by ``find()``
189 """Transform one instance of special syntax found by ``find()``
190
190
191 Takes a list of strings representing physical lines,
191 Takes a list of strings representing physical lines,
192 returns a similar list of transformed lines.
192 returns a similar list of transformed lines.
193 """
193 """
194 raise NotImplementedError
194 raise NotImplementedError
195
195
196 class MagicAssign(TokenTransformBase):
196 class MagicAssign(TokenTransformBase):
197 """Transformer for assignments from magics (a = %foo)"""
197 """Transformer for assignments from magics (a = %foo)"""
198 @classmethod
198 @classmethod
199 def find(cls, tokens_by_line):
199 def find(cls, tokens_by_line):
200 """Find the first magic assignment (a = %foo) in the cell.
200 """Find the first magic assignment (a = %foo) in the cell.
201 """
201 """
202 for line in tokens_by_line:
202 for line in tokens_by_line:
203 assign_ix = _find_assign_op(line)
203 assign_ix = _find_assign_op(line)
204 if (assign_ix is not None) \
204 if (assign_ix is not None) \
205 and (len(line) >= assign_ix + 2) \
205 and (len(line) >= assign_ix + 2) \
206 and (line[assign_ix+1].string == '%') \
206 and (line[assign_ix+1].string == '%') \
207 and (line[assign_ix+2].type == tokenize.NAME):
207 and (line[assign_ix+2].type == tokenize.NAME):
208 return cls(line[assign_ix+1].start)
208 return cls(line[assign_ix+1].start)
209
209
210 def transform(self, lines: List[str]):
210 def transform(self, lines: List[str]):
211 """Transform a magic assignment found by the ``find()`` classmethod.
211 """Transform a magic assignment found by the ``find()`` classmethod.
212 """
212 """
213 start_line, start_col = self.start_line, self.start_col
213 start_line, start_col = self.start_line, self.start_col
214 lhs = lines[start_line][:start_col]
214 lhs = lines[start_line][:start_col]
215 end_line = find_end_of_continued_line(lines, start_line)
215 end_line = find_end_of_continued_line(lines, start_line)
216 rhs = assemble_continued_line(lines, (start_line, start_col), end_line)
216 rhs = assemble_continued_line(lines, (start_line, start_col), end_line)
217 assert rhs.startswith('%'), rhs
217 assert rhs.startswith('%'), rhs
218 magic_name, _, args = rhs[1:].partition(' ')
218 magic_name, _, args = rhs[1:].partition(' ')
219
219
220 lines_before = lines[:start_line]
220 lines_before = lines[:start_line]
221 call = "get_ipython().run_line_magic({!r}, {!r})".format(magic_name, args)
221 call = "get_ipython().run_line_magic({!r}, {!r})".format(magic_name, args)
222 new_line = lhs + call + '\n'
222 new_line = lhs + call + '\n'
223 lines_after = lines[end_line+1:]
223 lines_after = lines[end_line+1:]
224
224
225 return lines_before + [new_line] + lines_after
225 return lines_before + [new_line] + lines_after
226
226
227
227
228 class SystemAssign(TokenTransformBase):
228 class SystemAssign(TokenTransformBase):
229 """Transformer for assignments from system commands (a = !foo)"""
229 """Transformer for assignments from system commands (a = !foo)"""
230 @classmethod
230 @classmethod
231 def find(cls, tokens_by_line):
231 def find(cls, tokens_by_line):
232 """Find the first system assignment (a = !foo) in the cell.
232 """Find the first system assignment (a = !foo) in the cell.
233 """
233 """
234 for line in tokens_by_line:
234 for line in tokens_by_line:
235 assign_ix = _find_assign_op(line)
235 assign_ix = _find_assign_op(line)
236 if (assign_ix is not None) \
236 if (assign_ix is not None) \
237 and not line[assign_ix].line.strip().startswith('=') \
237 and not line[assign_ix].line.strip().startswith('=') \
238 and (len(line) >= assign_ix + 2) \
238 and (len(line) >= assign_ix + 2) \
239 and (line[assign_ix + 1].type == tokenize.ERRORTOKEN):
239 and (line[assign_ix + 1].type == tokenize.ERRORTOKEN):
240 ix = assign_ix + 1
240 ix = assign_ix + 1
241
241
242 while ix < len(line) and line[ix].type == tokenize.ERRORTOKEN:
242 while ix < len(line) and line[ix].type == tokenize.ERRORTOKEN:
243 if line[ix].string == '!':
243 if line[ix].string == '!':
244 return cls(line[ix].start)
244 return cls(line[ix].start)
245 elif not line[ix].string.isspace():
245 elif not line[ix].string.isspace():
246 break
246 break
247 ix += 1
247 ix += 1
248
248
249 def transform(self, lines: List[str]):
249 def transform(self, lines: List[str]):
250 """Transform a system assignment found by the ``find()`` classmethod.
250 """Transform a system assignment found by the ``find()`` classmethod.
251 """
251 """
252 start_line, start_col = self.start_line, self.start_col
252 start_line, start_col = self.start_line, self.start_col
253
253
254 lhs = lines[start_line][:start_col]
254 lhs = lines[start_line][:start_col]
255 end_line = find_end_of_continued_line(lines, start_line)
255 end_line = find_end_of_continued_line(lines, start_line)
256 rhs = assemble_continued_line(lines, (start_line, start_col), end_line)
256 rhs = assemble_continued_line(lines, (start_line, start_col), end_line)
257 assert rhs.startswith('!'), rhs
257 assert rhs.startswith('!'), rhs
258 cmd = rhs[1:]
258 cmd = rhs[1:]
259
259
260 lines_before = lines[:start_line]
260 lines_before = lines[:start_line]
261 call = "get_ipython().getoutput({!r})".format(cmd)
261 call = "get_ipython().getoutput({!r})".format(cmd)
262 new_line = lhs + call + '\n'
262 new_line = lhs + call + '\n'
263 lines_after = lines[end_line + 1:]
263 lines_after = lines[end_line + 1:]
264
264
265 return lines_before + [new_line] + lines_after
265 return lines_before + [new_line] + lines_after
266
266
267 # The escape sequences that define the syntax transformations IPython will
267 # The escape sequences that define the syntax transformations IPython will
268 # apply to user input. These can NOT be just changed here: many regular
268 # apply to user input. These can NOT be just changed here: many regular
269 # expressions and other parts of the code may use their hardcoded values, and
269 # expressions and other parts of the code may use their hardcoded values, and
270 # for all intents and purposes they constitute the 'IPython syntax', so they
270 # for all intents and purposes they constitute the 'IPython syntax', so they
271 # should be considered fixed.
271 # should be considered fixed.
272
272
273 ESC_SHELL = '!' # Send line to underlying system shell
273 ESC_SHELL = '!' # Send line to underlying system shell
274 ESC_SH_CAP = '!!' # Send line to system shell and capture output
274 ESC_SH_CAP = '!!' # Send line to system shell and capture output
275 ESC_HELP = '?' # Find information about object
275 ESC_HELP = '?' # Find information about object
276 ESC_HELP2 = '??' # Find extra-detailed information about object
276 ESC_HELP2 = '??' # Find extra-detailed information about object
277 ESC_MAGIC = '%' # Call magic function
277 ESC_MAGIC = '%' # Call magic function
278 ESC_MAGIC2 = '%%' # Call cell-magic function
278 ESC_MAGIC2 = '%%' # Call cell-magic function
279 ESC_QUOTE = ',' # Split args on whitespace, quote each as string and call
279 ESC_QUOTE = ',' # Split args on whitespace, quote each as string and call
280 ESC_QUOTE2 = ';' # Quote all args as a single string, call
280 ESC_QUOTE2 = ';' # Quote all args as a single string, call
281 ESC_PAREN = '/' # Call first argument with rest of line as arguments
281 ESC_PAREN = '/' # Call first argument with rest of line as arguments
282
282
283 ESCAPE_SINGLES = {'!', '?', '%', ',', ';', '/'}
283 ESCAPE_SINGLES = {'!', '?', '%', ',', ';', '/'}
284 ESCAPE_DOUBLES = {'!!', '??'} # %% (cell magic) is handled separately
284 ESCAPE_DOUBLES = {'!!', '??'} # %% (cell magic) is handled separately
285
285
286 def _make_help_call(target, esc, next_input=None):
286 def _make_help_call(target, esc, next_input=None):
287 """Prepares a pinfo(2)/psearch call from a target name and the escape
287 """Prepares a pinfo(2)/psearch call from a target name and the escape
288 (i.e. ? or ??)"""
288 (i.e. ? or ??)"""
289 method = 'pinfo2' if esc == '??' \
289 method = 'pinfo2' if esc == '??' \
290 else 'psearch' if '*' in target \
290 else 'psearch' if '*' in target \
291 else 'pinfo'
291 else 'pinfo'
292 arg = " ".join([method, target])
292 arg = " ".join([method, target])
293 #Prepare arguments for get_ipython().run_line_magic(magic_name, magic_args)
293 #Prepare arguments for get_ipython().run_line_magic(magic_name, magic_args)
294 t_magic_name, _, t_magic_arg_s = arg.partition(' ')
294 t_magic_name, _, t_magic_arg_s = arg.partition(' ')
295 t_magic_name = t_magic_name.lstrip(ESC_MAGIC)
295 t_magic_name = t_magic_name.lstrip(ESC_MAGIC)
296 if next_input is None:
296 if next_input is None:
297 return 'get_ipython().run_line_magic(%r, %r)' % (t_magic_name, t_magic_arg_s)
297 return 'get_ipython().run_line_magic(%r, %r)' % (t_magic_name, t_magic_arg_s)
298 else:
298 else:
299 return 'get_ipython().set_next_input(%r);get_ipython().run_line_magic(%r, %r)' % \
299 return 'get_ipython().set_next_input(%r);get_ipython().run_line_magic(%r, %r)' % \
300 (next_input, t_magic_name, t_magic_arg_s)
300 (next_input, t_magic_name, t_magic_arg_s)
301
301
302 def _tr_help(content):
302 def _tr_help(content):
303 """Translate lines escaped with: ?
303 """Translate lines escaped with: ?
304
304
305 A naked help line should fire the intro help screen (shell.show_usage())
305 A naked help line should fire the intro help screen (shell.show_usage())
306 """
306 """
307 if not content:
307 if not content:
308 return 'get_ipython().show_usage()'
308 return 'get_ipython().show_usage()'
309
309
310 return _make_help_call(content, '?')
310 return _make_help_call(content, '?')
311
311
312 def _tr_help2(content):
312 def _tr_help2(content):
313 """Translate lines escaped with: ??
313 """Translate lines escaped with: ??
314
314
315 A naked help line should fire the intro help screen (shell.show_usage())
315 A naked help line should fire the intro help screen (shell.show_usage())
316 """
316 """
317 if not content:
317 if not content:
318 return 'get_ipython().show_usage()'
318 return 'get_ipython().show_usage()'
319
319
320 return _make_help_call(content, '??')
320 return _make_help_call(content, '??')
321
321
322 def _tr_magic(content):
322 def _tr_magic(content):
323 "Translate lines escaped with a percent sign: %"
323 "Translate lines escaped with a percent sign: %"
324 name, _, args = content.partition(' ')
324 name, _, args = content.partition(' ')
325 return 'get_ipython().run_line_magic(%r, %r)' % (name, args)
325 return 'get_ipython().run_line_magic(%r, %r)' % (name, args)
326
326
327 def _tr_quote(content):
327 def _tr_quote(content):
328 "Translate lines escaped with a comma: ,"
328 "Translate lines escaped with a comma: ,"
329 name, _, args = content.partition(' ')
329 name, _, args = content.partition(' ')
330 return '%s("%s")' % (name, '", "'.join(args.split()) )
330 return '%s("%s")' % (name, '", "'.join(args.split()) )
331
331
332 def _tr_quote2(content):
332 def _tr_quote2(content):
333 "Translate lines escaped with a semicolon: ;"
333 "Translate lines escaped with a semicolon: ;"
334 name, _, args = content.partition(' ')
334 name, _, args = content.partition(' ')
335 return '%s("%s")' % (name, args)
335 return '%s("%s")' % (name, args)
336
336
337 def _tr_paren(content):
337 def _tr_paren(content):
338 "Translate lines escaped with a slash: /"
338 "Translate lines escaped with a slash: /"
339 name, _, args = content.partition(' ')
339 name, _, args = content.partition(' ')
340 return '%s(%s)' % (name, ", ".join(args.split()))
340 return '%s(%s)' % (name, ", ".join(args.split()))
341
341
342 tr = { ESC_SHELL : 'get_ipython().system({!r})'.format,
342 tr = { ESC_SHELL : 'get_ipython().system({!r})'.format,
343 ESC_SH_CAP : 'get_ipython().getoutput({!r})'.format,
343 ESC_SH_CAP : 'get_ipython().getoutput({!r})'.format,
344 ESC_HELP : _tr_help,
344 ESC_HELP : _tr_help,
345 ESC_HELP2 : _tr_help2,
345 ESC_HELP2 : _tr_help2,
346 ESC_MAGIC : _tr_magic,
346 ESC_MAGIC : _tr_magic,
347 ESC_QUOTE : _tr_quote,
347 ESC_QUOTE : _tr_quote,
348 ESC_QUOTE2 : _tr_quote2,
348 ESC_QUOTE2 : _tr_quote2,
349 ESC_PAREN : _tr_paren }
349 ESC_PAREN : _tr_paren }
350
350
351 class EscapedCommand(TokenTransformBase):
351 class EscapedCommand(TokenTransformBase):
352 """Transformer for escaped commands like %foo, !foo, or /foo"""
352 """Transformer for escaped commands like %foo, !foo, or /foo"""
353 @classmethod
353 @classmethod
354 def find(cls, tokens_by_line):
354 def find(cls, tokens_by_line):
355 """Find the first escaped command (%foo, !foo, etc.) in the cell.
355 """Find the first escaped command (%foo, !foo, etc.) in the cell.
356 """
356 """
357 for line in tokens_by_line:
357 for line in tokens_by_line:
358 ix = 0
358 ix = 0
359 while line[ix].type in {tokenize.INDENT, tokenize.DEDENT}:
359 while line[ix].type in {tokenize.INDENT, tokenize.DEDENT}:
360 ix += 1
360 ix += 1
361 if line[ix].string in ESCAPE_SINGLES:
361 if line[ix].string in ESCAPE_SINGLES:
362 return cls(line[ix].start)
362 return cls(line[ix].start)
363
363
364 def transform(self, lines):
364 def transform(self, lines):
365 """Transform an escaped line found by the ``find()`` classmethod.
365 """Transform an escaped line found by the ``find()`` classmethod.
366 """
366 """
367 start_line, start_col = self.start_line, self.start_col
367 start_line, start_col = self.start_line, self.start_col
368
368
369 indent = lines[start_line][:start_col]
369 indent = lines[start_line][:start_col]
370 end_line = find_end_of_continued_line(lines, start_line)
370 end_line = find_end_of_continued_line(lines, start_line)
371 line = assemble_continued_line(lines, (start_line, start_col), end_line)
371 line = assemble_continued_line(lines, (start_line, start_col), end_line)
372
372
373 if len(line) > 1 and line[:2] in ESCAPE_DOUBLES:
373 if len(line) > 1 and line[:2] in ESCAPE_DOUBLES:
374 escape, content = line[:2], line[2:]
374 escape, content = line[:2], line[2:]
375 else:
375 else:
376 escape, content = line[:1], line[1:]
376 escape, content = line[:1], line[1:]
377
377
378 if escape in tr:
378 if escape in tr:
379 call = tr[escape](content)
379 call = tr[escape](content)
380 else:
380 else:
381 call = ''
381 call = ''
382
382
383 lines_before = lines[:start_line]
383 lines_before = lines[:start_line]
384 new_line = indent + call + '\n'
384 new_line = indent + call + '\n'
385 lines_after = lines[end_line + 1:]
385 lines_after = lines[end_line + 1:]
386
386
387 return lines_before + [new_line] + lines_after
387 return lines_before + [new_line] + lines_after
388
388
389 _help_end_re = re.compile(r"""(%{0,2}
389 _help_end_re = re.compile(r"""(%{0,2}
390 [a-zA-Z_*][\w*]* # Variable name
390 [a-zA-Z_*][\w*]* # Variable name
391 (\.[a-zA-Z_*][\w*]*)* # .etc.etc
391 (\.[a-zA-Z_*][\w*]*)* # .etc.etc
392 )
392 )
393 (\?\??)$ # ? or ??
393 (\?\??)$ # ? or ??
394 """,
394 """,
395 re.VERBOSE)
395 re.VERBOSE)
396
396
397 class HelpEnd(TokenTransformBase):
397 class HelpEnd(TokenTransformBase):
398 """Transformer for help syntax: obj? and obj??"""
398 """Transformer for help syntax: obj? and obj??"""
399 # This needs to be higher priority (lower number) than EscapedCommand so
399 # This needs to be higher priority (lower number) than EscapedCommand so
400 # that inspecting magics (%foo?) works.
400 # that inspecting magics (%foo?) works.
401 priority = 5
401 priority = 5
402
402
403 def __init__(self, start, q_locn):
403 def __init__(self, start, q_locn):
404 super().__init__(start)
404 super().__init__(start)
405 self.q_line = q_locn[0] - 1 # Shift from 1-indexed to 0-indexed
405 self.q_line = q_locn[0] - 1 # Shift from 1-indexed to 0-indexed
406 self.q_col = q_locn[1]
406 self.q_col = q_locn[1]
407
407
408 @classmethod
408 @classmethod
409 def find(cls, tokens_by_line):
409 def find(cls, tokens_by_line):
410 """Find the first help command (foo?) in the cell.
410 """Find the first help command (foo?) in the cell.
411 """
411 """
412 for line in tokens_by_line:
412 for line in tokens_by_line:
413 # Last token is NEWLINE; look at last but one
413 # Last token is NEWLINE; look at last but one
414 if len(line) > 2 and line[-2].string == '?':
414 if len(line) > 2 and line[-2].string == '?':
415 # Find the first token that's not INDENT/DEDENT
415 # Find the first token that's not INDENT/DEDENT
416 ix = 0
416 ix = 0
417 while line[ix].type in {tokenize.INDENT, tokenize.DEDENT}:
417 while line[ix].type in {tokenize.INDENT, tokenize.DEDENT}:
418 ix += 1
418 ix += 1
419 return cls(line[ix].start, line[-2].start)
419 return cls(line[ix].start, line[-2].start)
420
420
421 def transform(self, lines):
421 def transform(self, lines):
422 """Transform a help command found by the ``find()`` classmethod.
422 """Transform a help command found by the ``find()`` classmethod.
423 """
423 """
424 piece = ''.join(lines[self.start_line:self.q_line+1])
424 piece = ''.join(lines[self.start_line:self.q_line+1])
425 indent, content = piece[:self.start_col], piece[self.start_col:]
425 indent, content = piece[:self.start_col], piece[self.start_col:]
426 lines_before = lines[:self.start_line]
426 lines_before = lines[:self.start_line]
427 lines_after = lines[self.q_line + 1:]
427 lines_after = lines[self.q_line + 1:]
428
428
429 m = _help_end_re.search(content)
429 m = _help_end_re.search(content)
430 if not m:
430 if not m:
431 raise SyntaxError(content)
431 raise SyntaxError(content)
432 assert m is not None, content
432 assert m is not None, content
433 target = m.group(1)
433 target = m.group(1)
434 esc = m.group(3)
434 esc = m.group(3)
435
435
436 # If we're mid-command, put it back on the next prompt for the user.
436 # If we're mid-command, put it back on the next prompt for the user.
437 next_input = None
437 next_input = None
438 if (not lines_before) and (not lines_after) \
438 if (not lines_before) and (not lines_after) \
439 and content.strip() != m.group(0):
439 and content.strip() != m.group(0):
440 next_input = content.rstrip('?\n')
440 next_input = content.rstrip('?\n')
441
441
442 call = _make_help_call(target, esc, next_input=next_input)
442 call = _make_help_call(target, esc, next_input=next_input)
443 new_line = indent + call + '\n'
443 new_line = indent + call + '\n'
444
444
445 return lines_before + [new_line] + lines_after
445 return lines_before + [new_line] + lines_after
446
446
447 def make_tokens_by_line(lines):
447 def make_tokens_by_line(lines):
448 """Tokenize a series of lines and group tokens by line.
448 """Tokenize a series of lines and group tokens by line.
449
449
450 The tokens for a multiline Python string or expression are
450 The tokens for a multiline Python string or expression are
451 grouped as one line.
451 grouped as one line.
452 """
452 """
453 # NL tokens are used inside multiline expressions, but also after blank
453 # NL tokens are used inside multiline expressions, but also after blank
454 # lines or comments. This is intentional - see https://bugs.python.org/issue17061
454 # lines or comments. This is intentional - see https://bugs.python.org/issue17061
455 # We want to group the former case together but split the latter, so we
455 # We want to group the former case together but split the latter, so we
456 # track parentheses level, similar to the internals of tokenize.
456 # track parentheses level, similar to the internals of tokenize.
457 NEWLINE, NL = tokenize.NEWLINE, tokenize.NL
457 NEWLINE, NL = tokenize.NEWLINE, tokenize.NL
458 tokens_by_line = [[]]
458 tokens_by_line = [[]]
459 parenlev = 0
459 parenlev = 0
460 try:
460 try:
461 for token in tokenize.generate_tokens(iter(lines).__next__):
461 for token in tokenize.generate_tokens(iter(lines).__next__):
462 tokens_by_line[-1].append(token)
462 tokens_by_line[-1].append(token)
463 if (token.type == NEWLINE) \
463 if (token.type == NEWLINE) \
464 or ((token.type == NL) and (parenlev <= 0)):
464 or ((token.type == NL) and (parenlev <= 0)):
465 tokens_by_line.append([])
465 tokens_by_line.append([])
466 elif token.string in {'(', '[', '{'}:
466 elif token.string in {'(', '[', '{'}:
467 parenlev += 1
467 parenlev += 1
468 elif token.string in {')', ']', '}'}:
468 elif token.string in {')', ']', '}'}:
469 if parenlev > 0:
469 if parenlev > 0:
470 parenlev -= 1
470 parenlev -= 1
471 except tokenize.TokenError:
471 except tokenize.TokenError:
472 # Input ended in a multiline string or expression. That's OK for us.
472 # Input ended in a multiline string or expression. That's OK for us.
473 pass
473 pass
474
474
475
475
476 if not tokens_by_line[-1]:
476 if not tokens_by_line[-1]:
477 tokens_by_line.pop()
477 tokens_by_line.pop()
478
478
479
479
480 return tokens_by_line
480 return tokens_by_line
481
481
482 def show_linewise_tokens(s: str):
482 def show_linewise_tokens(s: str):
483 """For investigation and debugging"""
483 """For investigation and debugging"""
484 if not s.endswith('\n'):
484 if not s.endswith('\n'):
485 s += '\n'
485 s += '\n'
486 lines = s.splitlines(keepends=True)
486 lines = s.splitlines(keepends=True)
487 for line in make_tokens_by_line(lines):
487 for line in make_tokens_by_line(lines):
488 print("Line -------")
488 print("Line -------")
489 for tokinfo in line:
489 for tokinfo in line:
490 print(" ", tokinfo)
490 print(" ", tokinfo)
491
491
492 # Arbitrary limit to prevent getting stuck in infinite loops
492 # Arbitrary limit to prevent getting stuck in infinite loops
493 TRANSFORM_LOOP_LIMIT = 500
493 TRANSFORM_LOOP_LIMIT = 500
494
494
495 class TransformerManager:
495 class TransformerManager:
496 """Applies various transformations to a cell or code block.
496 """Applies various transformations to a cell or code block.
497
497
498 The key methods for external use are ``transform_cell()``
498 The key methods for external use are ``transform_cell()``
499 and ``check_complete()``.
499 and ``check_complete()``.
500 """
500 """
501 def __init__(self):
501 def __init__(self):
502 self.cleanup_transforms = [
502 self.cleanup_transforms = [
503 leading_indent,
503 leading_indent,
504 classic_prompt,
504 classic_prompt,
505 ipython_prompt,
505 ipython_prompt,
506 ]
506 ]
507 self.line_transforms = [
507 self.line_transforms = [
508 cell_magic,
508 cell_magic,
509 ]
509 ]
510 self.token_transformers = [
510 self.token_transformers = [
511 MagicAssign,
511 MagicAssign,
512 SystemAssign,
512 SystemAssign,
513 EscapedCommand,
513 EscapedCommand,
514 HelpEnd,
514 HelpEnd,
515 ]
515 ]
516
516
517 def do_one_token_transform(self, lines):
517 def do_one_token_transform(self, lines):
518 """Find and run the transform earliest in the code.
518 """Find and run the transform earliest in the code.
519
519
520 Returns (changed, lines).
520 Returns (changed, lines).
521
521
522 This method is called repeatedly until changed is False, indicating
522 This method is called repeatedly until changed is False, indicating
523 that all available transformations are complete.
523 that all available transformations are complete.
524
524
525 The tokens following IPython special syntax might not be valid, so
525 The tokens following IPython special syntax might not be valid, so
526 the transformed code is retokenised every time to identify the next
526 the transformed code is retokenised every time to identify the next
527 piece of special syntax. Hopefully long code cells are mostly valid
527 piece of special syntax. Hopefully long code cells are mostly valid
528 Python, not using lots of IPython special syntax, so this shouldn't be
528 Python, not using lots of IPython special syntax, so this shouldn't be
529 a performance issue.
529 a performance issue.
530 """
530 """
531 tokens_by_line = make_tokens_by_line(lines)
531 tokens_by_line = make_tokens_by_line(lines)
532 candidates = []
532 candidates = []
533 for transformer_cls in self.token_transformers:
533 for transformer_cls in self.token_transformers:
534 transformer = transformer_cls.find(tokens_by_line)
534 transformer = transformer_cls.find(tokens_by_line)
535 if transformer:
535 if transformer:
536 candidates.append(transformer)
536 candidates.append(transformer)
537
537
538 if not candidates:
538 if not candidates:
539 # Nothing to transform
539 # Nothing to transform
540 return False, lines
540 return False, lines
541 ordered_transformers = sorted(candidates, key=TokenTransformBase.sortby)
541 ordered_transformers = sorted(candidates, key=TokenTransformBase.sortby)
542 for transformer in ordered_transformers:
542 for transformer in ordered_transformers:
543 try:
543 try:
544 return True, transformer.transform(lines)
544 return True, transformer.transform(lines)
545 except SyntaxError:
545 except SyntaxError:
546 pass
546 pass
547 return False, lines
547 return False, lines
548
548
549 def do_token_transforms(self, lines):
549 def do_token_transforms(self, lines):
550 for _ in range(TRANSFORM_LOOP_LIMIT):
550 for _ in range(TRANSFORM_LOOP_LIMIT):
551 changed, lines = self.do_one_token_transform(lines)
551 changed, lines = self.do_one_token_transform(lines)
552 if not changed:
552 if not changed:
553 return lines
553 return lines
554
554
555 raise RuntimeError("Input transformation still changing after "
555 raise RuntimeError("Input transformation still changing after "
556 "%d iterations. Aborting." % TRANSFORM_LOOP_LIMIT)
556 "%d iterations. Aborting." % TRANSFORM_LOOP_LIMIT)
557
557
558 def transform_cell(self, cell: str) -> str:
558 def transform_cell(self, cell: str) -> str:
559 """Transforms a cell of input code"""
559 """Transforms a cell of input code"""
560 if not cell.endswith('\n'):
560 if not cell.endswith('\n'):
561 cell += '\n' # Ensure the cell has a trailing newline
561 cell += '\n' # Ensure the cell has a trailing newline
562 lines = cell.splitlines(keepends=True)
562 lines = cell.splitlines(keepends=True)
563 for transform in self.cleanup_transforms + self.line_transforms:
563 for transform in self.cleanup_transforms + self.line_transforms:
564 lines = transform(lines)
564 lines = transform(lines)
565
565
566 lines = self.do_token_transforms(lines)
566 lines = self.do_token_transforms(lines)
567 return ''.join(lines)
567 return ''.join(lines)
568
568
569 def check_complete(self, cell: str):
569 def check_complete(self, cell: str):
570 """Return whether a block of code is ready to execute, or should be continued
570 """Return whether a block of code is ready to execute, or should be continued
571
571
572 Parameters
572 Parameters
573 ----------
573 ----------
574 source : string
574 source : string
575 Python input code, which can be multiline.
575 Python input code, which can be multiline.
576
576
577 Returns
577 Returns
578 -------
578 -------
579 status : str
579 status : str
580 One of 'complete', 'incomplete', or 'invalid' if source is not a
580 One of 'complete', 'incomplete', or 'invalid' if source is not a
581 prefix of valid code.
581 prefix of valid code.
582 indent_spaces : int or None
582 indent_spaces : int or None
583 The number of spaces by which to indent the next line of code. If
583 The number of spaces by which to indent the next line of code. If
584 status is not 'incomplete', this is None.
584 status is not 'incomplete', this is None.
585 """
585 """
586 # Remember if the lines ends in a new line.
586 # Remember if the lines ends in a new line.
587 ends_with_newline = False
587 ends_with_newline = False
588 for character in reversed(cell):
588 for character in reversed(cell):
589 if character == '\n':
589 if character == '\n':
590 ends_with_newline = True
590 ends_with_newline = True
591 break
591 break
592 elif character.strip():
592 elif character.strip():
593 break
593 break
594 else:
594 else:
595 continue
595 continue
596
596
597 if ends_with_newline:
597 if ends_with_newline:
598 # Append an newline for consistent tokenization
598 # Append an newline for consistent tokenization
599 # See https://bugs.python.org/issue33899
599 # See https://bugs.python.org/issue33899
600 cell += '\n'
600 cell += '\n'
601
601
602 lines = cell.splitlines(keepends=True)
602 lines = cell.splitlines(keepends=True)
603
603
604 if not lines:
604 if not lines:
605 return 'complete', None
605 return 'complete', None
606
606
607 if lines[-1].endswith('\\'):
607 if lines[-1].endswith('\\'):
608 # Explicit backslash continuation
608 # Explicit backslash continuation
609 return 'incomplete', find_last_indent(lines)
609 return 'incomplete', find_last_indent(lines)
610
610
611 try:
611 try:
612 for transform in self.cleanup_transforms:
612 for transform in self.cleanup_transforms:
613 lines = transform(lines)
613 lines = transform(lines)
614 except SyntaxError:
614 except SyntaxError:
615 return 'invalid', None
615 return 'invalid', None
616
616
617 if lines[0].startswith('%%'):
617 if lines[0].startswith('%%'):
618 # Special case for cell magics - completion marked by blank line
618 # Special case for cell magics - completion marked by blank line
619 if lines[-1].strip():
619 if lines[-1].strip():
620 return 'incomplete', find_last_indent(lines)
620 return 'incomplete', find_last_indent(lines)
621 else:
621 else:
622 return 'complete', None
622 return 'complete', None
623
623
624 try:
624 try:
625 for transform in self.line_transforms:
625 for transform in self.line_transforms:
626 lines = transform(lines)
626 lines = transform(lines)
627 lines = self.do_token_transforms(lines)
627 lines = self.do_token_transforms(lines)
628 except SyntaxError:
628 except SyntaxError:
629 return 'invalid', None
629 return 'invalid', None
630
630
631 tokens_by_line = make_tokens_by_line(lines)
631 tokens_by_line = make_tokens_by_line(lines)
632
632
633 if not tokens_by_line:
633 if not tokens_by_line:
634 return 'incomplete', find_last_indent(lines)
634 return 'incomplete', find_last_indent(lines)
635
635
636 if tokens_by_line[-1][-1].type != tokenize.ENDMARKER:
636 if tokens_by_line[-1][-1].type != tokenize.ENDMARKER:
637 # We're in a multiline string or expression
637 # We're in a multiline string or expression
638 return 'incomplete', find_last_indent(lines)
638 return 'incomplete', find_last_indent(lines)
639
639
640 newline_types = {tokenize.NEWLINE, tokenize.COMMENT, tokenize.ENDMARKER}
640 newline_types = {tokenize.NEWLINE, tokenize.COMMENT, tokenize.ENDMARKER}
641
641
642 # Remove newline_types for the list of tokens
642 # Remove newline_types for the list of tokens
643 while len(tokens_by_line) > 1 and len(tokens_by_line[-1]) == 1 \
643 while len(tokens_by_line) > 1 and len(tokens_by_line[-1]) == 1 \
644 and tokens_by_line[-1][-1].type in newline_types:
644 and tokens_by_line[-1][-1].type in newline_types:
645 tokens_by_line.pop()
645 tokens_by_line.pop()
646
646
647
648 while tokens_by_line[-1] and tokens_by_line[-1][-1].type in newline_types:
647 while tokens_by_line[-1] and tokens_by_line[-1][-1].type in newline_types:
649 tokens_by_line[-1].pop()
648 tokens_by_line[-1].pop()
650
649
651 if len(tokens_by_line) == 1 and not tokens_by_line[-1]:
650 if len(tokens_by_line) == 1 and not tokens_by_line[-1]:
652 return 'incomplete', 0
651 return 'incomplete', 0
653
652
654 if tokens_by_line[-1][-1].string == ':':
653 if tokens_by_line[-1][-1].string == ':':
655 # The last line starts a block (e.g. 'if foo:')
654 # The last line starts a block (e.g. 'if foo:')
656 ix = 0
655 ix = 0
657 while tokens_by_line[-1][ix].type in {tokenize.INDENT, tokenize.DEDENT}:
656 while tokens_by_line[-1][ix].type in {tokenize.INDENT, tokenize.DEDENT}:
658 ix += 1
657 ix += 1
659
658
660 indent = tokens_by_line[-1][ix].start[1]
659 indent = tokens_by_line[-1][ix].start[1]
661 return 'incomplete', indent + 4
660 return 'incomplete', indent + 4
662
661
663 if tokens_by_line[-1][0].line.endswith('\\'):
662 if tokens_by_line[-1][0].line.endswith('\\'):
664 return 'incomplete', None
663 return 'incomplete', None
665
664
666 # At this point, our checks think the code is complete (or invalid).
665 # At this point, our checks think the code is complete (or invalid).
667 # We'll use codeop.compile_command to check this with the real parser
666 # We'll use codeop.compile_command to check this with the real parser
668 try:
667 try:
669 with warnings.catch_warnings():
668 with warnings.catch_warnings():
670 warnings.simplefilter('error', SyntaxWarning)
669 warnings.simplefilter('error', SyntaxWarning)
671 res = compile_command(''.join(lines), symbol='exec')
670 res = compile_command(''.join(lines), symbol='exec')
672 except (SyntaxError, OverflowError, ValueError, TypeError,
671 except (SyntaxError, OverflowError, ValueError, TypeError,
673 MemoryError, SyntaxWarning):
672 MemoryError, SyntaxWarning):
674 return 'invalid', None
673 return 'invalid', None
675 else:
674 else:
676 if res is None:
675 if res is None:
677 return 'incomplete', find_last_indent(lines)
676 return 'incomplete', find_last_indent(lines)
678
677
679 if tokens_by_line[-1][-1].type == tokenize.DEDENT:
678 if tokens_by_line[-1][-1].type == tokenize.DEDENT:
680 if ends_with_newline:
679 if ends_with_newline:
681 return 'complete', None
680 return 'complete', None
682 return 'incomplete', find_last_indent(lines)
681 return 'incomplete', find_last_indent(lines)
683
682
684 if len(tokens_by_line[-1]) <= 1:
685 return 'incomplete', find_last_indent(lines)
686
687 # If there's a blank line at the end, assume we're ready to execute
683 # If there's a blank line at the end, assume we're ready to execute
688 if not lines[-1].strip():
684 if not lines[-1].strip():
689 return 'complete', None
685 return 'complete', None
690
686
691 return 'complete', None
687 return 'complete', None
692
688
693
689
694 def find_last_indent(lines):
690 def find_last_indent(lines):
695 m = _indent_re.match(lines[-1])
691 m = _indent_re.match(lines[-1])
696 if not m:
692 if not m:
697 return 0
693 return 0
698 return len(m.group(0).replace('\t', ' '*4))
694 return len(m.group(0).replace('\t', ' '*4))
General Comments 0
You need to be logged in to leave comments. Login now