##// END OF EJS Templates
support for unicode identifiers...
Markus Wageringel -
Show More
@@ -1,354 +1,354 b''
1 1 # encoding: utf-8
2 2 """Implementations for various useful completers.
3 3
4 4 These are all loaded by default by IPython.
5 5 """
6 6 #-----------------------------------------------------------------------------
7 7 # Copyright (C) 2010-2011 The IPython Development Team.
8 8 #
9 9 # Distributed under the terms of the BSD License.
10 10 #
11 11 # The full license is in the file COPYING.txt, distributed with this software.
12 12 #-----------------------------------------------------------------------------
13 13
14 14 #-----------------------------------------------------------------------------
15 15 # Imports
16 16 #-----------------------------------------------------------------------------
17 17
18 18 # Stdlib imports
19 19 import glob
20 20 import inspect
21 21 import os
22 22 import re
23 23 import sys
24 24 from importlib import import_module
25 25 from importlib.machinery import all_suffixes
26 26
27 27
28 28 # Third-party imports
29 29 from time import time
30 30 from zipimport import zipimporter
31 31
32 32 # Our own imports
33 33 from .completer import expand_user, compress_user
34 34 from .error import TryNext
35 35 from ..utils._process_common import arg_split
36 36
37 37 # FIXME: this should be pulled in with the right call via the component system
38 38 from IPython import get_ipython
39 39
40 40 from typing import List
41 41
42 42 #-----------------------------------------------------------------------------
43 43 # Globals and constants
44 44 #-----------------------------------------------------------------------------
45 45 _suffixes = all_suffixes()
46 46
47 47 # Time in seconds after which the rootmodules will be stored permanently in the
48 48 # ipython ip.db database (kept in the user's .ipython dir).
49 49 TIMEOUT_STORAGE = 2
50 50
51 51 # Time in seconds after which we give up
52 52 TIMEOUT_GIVEUP = 20
53 53
54 54 # Regular expression for the python import statement
55 import_re = re.compile(r'(?P<name>[a-zA-Z_][a-zA-Z0-9_]*?)'
55 import_re = re.compile(r'(?P<name>[^\W\d]\w*?)'
56 56 r'(?P<package>[/\\]__init__)?'
57 57 r'(?P<suffix>%s)$' %
58 58 r'|'.join(re.escape(s) for s in _suffixes))
59 59
60 60 # RE for the ipython %run command (python + ipython scripts)
61 61 magic_run_re = re.compile(r'.*(\.ipy|\.ipynb|\.py[w]?)$')
62 62
63 63 #-----------------------------------------------------------------------------
64 64 # Local utilities
65 65 #-----------------------------------------------------------------------------
66 66
67 67 def module_list(path):
68 68 """
69 69 Return the list containing the names of the modules available in the given
70 70 folder.
71 71 """
72 72 # sys.path has the cwd as an empty string, but isdir/listdir need it as '.'
73 73 if path == '':
74 74 path = '.'
75 75
76 76 # A few local constants to be used in loops below
77 77 pjoin = os.path.join
78 78
79 79 if os.path.isdir(path):
80 80 # Build a list of all files in the directory and all files
81 81 # in its subdirectories. For performance reasons, do not
82 82 # recurse more than one level into subdirectories.
83 83 files = []
84 84 for root, dirs, nondirs in os.walk(path, followlinks=True):
85 85 subdir = root[len(path)+1:]
86 86 if subdir:
87 87 files.extend(pjoin(subdir, f) for f in nondirs)
88 88 dirs[:] = [] # Do not recurse into additional subdirectories.
89 89 else:
90 90 files.extend(nondirs)
91 91
92 92 else:
93 93 try:
94 94 files = list(zipimporter(path)._files.keys())
95 95 except:
96 96 files = []
97 97
98 98 # Build a list of modules which match the import_re regex.
99 99 modules = []
100 100 for f in files:
101 101 m = import_re.match(f)
102 102 if m:
103 103 modules.append(m.group('name'))
104 104 return list(set(modules))
105 105
106 106
107 107 def get_root_modules():
108 108 """
109 109 Returns a list containing the names of all the modules available in the
110 110 folders of the pythonpath.
111 111
112 112 ip.db['rootmodules_cache'] maps sys.path entries to list of modules.
113 113 """
114 114 ip = get_ipython()
115 115 if ip is None:
116 116 # No global shell instance to store cached list of modules.
117 117 # Don't try to scan for modules every time.
118 118 return list(sys.builtin_module_names)
119 119
120 120 rootmodules_cache = ip.db.get('rootmodules_cache', {})
121 121 rootmodules = list(sys.builtin_module_names)
122 122 start_time = time()
123 123 store = False
124 124 for path in sys.path:
125 125 try:
126 126 modules = rootmodules_cache[path]
127 127 except KeyError:
128 128 modules = module_list(path)
129 129 try:
130 130 modules.remove('__init__')
131 131 except ValueError:
132 132 pass
133 133 if path not in ('', '.'): # cwd modules should not be cached
134 134 rootmodules_cache[path] = modules
135 135 if time() - start_time > TIMEOUT_STORAGE and not store:
136 136 store = True
137 137 print("\nCaching the list of root modules, please wait!")
138 138 print("(This will only be done once - type '%rehashx' to "
139 139 "reset cache!)\n")
140 140 sys.stdout.flush()
141 141 if time() - start_time > TIMEOUT_GIVEUP:
142 142 print("This is taking too long, we give up.\n")
143 143 return []
144 144 rootmodules.extend(modules)
145 145 if store:
146 146 ip.db['rootmodules_cache'] = rootmodules_cache
147 147 rootmodules = list(set(rootmodules))
148 148 return rootmodules
149 149
150 150
151 151 def is_importable(module, attr, only_modules):
152 152 if only_modules:
153 153 return inspect.ismodule(getattr(module, attr))
154 154 else:
155 155 return not(attr[:2] == '__' and attr[-2:] == '__')
156 156
157 157
158 158 def try_import(mod: str, only_modules=False) -> List[str]:
159 159 """
160 160 Try to import given module and return list of potential completions.
161 161 """
162 162 mod = mod.rstrip('.')
163 163 try:
164 164 m = import_module(mod)
165 165 except:
166 166 return []
167 167
168 168 m_is_init = '__init__' in (getattr(m, '__file__', '') or '')
169 169
170 170 completions = []
171 171 if (not hasattr(m, '__file__')) or (not only_modules) or m_is_init:
172 172 completions.extend( [attr for attr in dir(m) if
173 173 is_importable(m, attr, only_modules)])
174 174
175 175 completions.extend(getattr(m, '__all__', []))
176 176 if m_is_init:
177 177 completions.extend(module_list(os.path.dirname(m.__file__)))
178 178 completions_set = {c for c in completions if isinstance(c, str)}
179 179 completions_set.discard('__init__')
180 180 return list(completions_set)
181 181
182 182
183 183 #-----------------------------------------------------------------------------
184 184 # Completion-related functions.
185 185 #-----------------------------------------------------------------------------
186 186
187 187 def quick_completer(cmd, completions):
188 188 r""" Easily create a trivial completer for a command.
189 189
190 190 Takes either a list of completions, or all completions in string (that will
191 191 be split on whitespace).
192 192
193 193 Example::
194 194
195 195 [d:\ipython]|1> import ipy_completers
196 196 [d:\ipython]|2> ipy_completers.quick_completer('foo', ['bar','baz'])
197 197 [d:\ipython]|3> foo b<TAB>
198 198 bar baz
199 199 [d:\ipython]|3> foo ba
200 200 """
201 201
202 202 if isinstance(completions, str):
203 203 completions = completions.split()
204 204
205 205 def do_complete(self, event):
206 206 return completions
207 207
208 208 get_ipython().set_hook('complete_command',do_complete, str_key = cmd)
209 209
210 210 def module_completion(line):
211 211 """
212 212 Returns a list containing the completion possibilities for an import line.
213 213
214 214 The line looks like this :
215 215 'import xml.d'
216 216 'from xml.dom import'
217 217 """
218 218
219 219 words = line.split(' ')
220 220 nwords = len(words)
221 221
222 222 # from whatever <tab> -> 'import '
223 223 if nwords == 3 and words[0] == 'from':
224 224 return ['import ']
225 225
226 226 # 'from xy<tab>' or 'import xy<tab>'
227 227 if nwords < 3 and (words[0] in {'%aimport', 'import', 'from'}) :
228 228 if nwords == 1:
229 229 return get_root_modules()
230 230 mod = words[1].split('.')
231 231 if len(mod) < 2:
232 232 return get_root_modules()
233 233 completion_list = try_import('.'.join(mod[:-1]), True)
234 234 return ['.'.join(mod[:-1] + [el]) for el in completion_list]
235 235
236 236 # 'from xyz import abc<tab>'
237 237 if nwords >= 3 and words[0] == 'from':
238 238 mod = words[1]
239 239 return try_import(mod)
240 240
241 241 #-----------------------------------------------------------------------------
242 242 # Completers
243 243 #-----------------------------------------------------------------------------
244 244 # These all have the func(self, event) signature to be used as custom
245 245 # completers
246 246
247 247 def module_completer(self,event):
248 248 """Give completions after user has typed 'import ...' or 'from ...'"""
249 249
250 250 # This works in all versions of python. While 2.5 has
251 251 # pkgutil.walk_packages(), that particular routine is fairly dangerous,
252 252 # since it imports *EVERYTHING* on sys.path. That is: a) very slow b) full
253 253 # of possibly problematic side effects.
254 254 # This search the folders in the sys.path for available modules.
255 255
256 256 return module_completion(event.line)
257 257
258 258 # FIXME: there's a lot of logic common to the run, cd and builtin file
259 259 # completers, that is currently reimplemented in each.
260 260
261 261 def magic_run_completer(self, event):
262 262 """Complete files that end in .py or .ipy or .ipynb for the %run command.
263 263 """
264 264 comps = arg_split(event.line, strict=False)
265 265 # relpath should be the current token that we need to complete.
266 266 if (len(comps) > 1) and (not event.line.endswith(' ')):
267 267 relpath = comps[-1].strip("'\"")
268 268 else:
269 269 relpath = ''
270 270
271 271 #print("\nev=", event) # dbg
272 272 #print("rp=", relpath) # dbg
273 273 #print('comps=', comps) # dbg
274 274
275 275 lglob = glob.glob
276 276 isdir = os.path.isdir
277 277 relpath, tilde_expand, tilde_val = expand_user(relpath)
278 278
279 279 # Find if the user has already typed the first filename, after which we
280 280 # should complete on all files, since after the first one other files may
281 281 # be arguments to the input script.
282 282
283 283 if any(magic_run_re.match(c) for c in comps):
284 284 matches = [f.replace('\\','/') + ('/' if isdir(f) else '')
285 285 for f in lglob(relpath+'*')]
286 286 else:
287 287 dirs = [f.replace('\\','/') + "/" for f in lglob(relpath+'*') if isdir(f)]
288 288 pys = [f.replace('\\','/')
289 289 for f in lglob(relpath+'*.py') + lglob(relpath+'*.ipy') +
290 290 lglob(relpath+'*.ipynb') + lglob(relpath + '*.pyw')]
291 291
292 292 matches = dirs + pys
293 293
294 294 #print('run comp:', dirs+pys) # dbg
295 295 return [compress_user(p, tilde_expand, tilde_val) for p in matches]
296 296
297 297
298 298 def cd_completer(self, event):
299 299 """Completer function for cd, which only returns directories."""
300 300 ip = get_ipython()
301 301 relpath = event.symbol
302 302
303 303 #print(event) # dbg
304 304 if event.line.endswith('-b') or ' -b ' in event.line:
305 305 # return only bookmark completions
306 306 bkms = self.db.get('bookmarks', None)
307 307 if bkms:
308 308 return bkms.keys()
309 309 else:
310 310 return []
311 311
312 312 if event.symbol == '-':
313 313 width_dh = str(len(str(len(ip.user_ns['_dh']) + 1)))
314 314 # jump in directory history by number
315 315 fmt = '-%0' + width_dh +'d [%s]'
316 316 ents = [ fmt % (i,s) for i,s in enumerate(ip.user_ns['_dh'])]
317 317 if len(ents) > 1:
318 318 return ents
319 319 return []
320 320
321 321 if event.symbol.startswith('--'):
322 322 return ["--" + os.path.basename(d) for d in ip.user_ns['_dh']]
323 323
324 324 # Expand ~ in path and normalize directory separators.
325 325 relpath, tilde_expand, tilde_val = expand_user(relpath)
326 326 relpath = relpath.replace('\\','/')
327 327
328 328 found = []
329 329 for d in [f.replace('\\','/') + '/' for f in glob.glob(relpath+'*')
330 330 if os.path.isdir(f)]:
331 331 if ' ' in d:
332 332 # we don't want to deal with any of that, complex code
333 333 # for this is elsewhere
334 334 raise TryNext
335 335
336 336 found.append(d)
337 337
338 338 if not found:
339 339 if os.path.isdir(relpath):
340 340 return [compress_user(relpath, tilde_expand, tilde_val)]
341 341
342 342 # if no completions so far, try bookmarks
343 343 bks = self.db.get('bookmarks',{})
344 344 bkmatches = [s for s in bks if s.startswith(event.symbol)]
345 345 if bkmatches:
346 346 return bkmatches
347 347
348 348 raise TryNext
349 349
350 350 return [compress_user(p, tilde_expand, tilde_val) for p in found]
351 351
352 352 def reset_completer(self, event):
353 353 "A completer for %reset magic"
354 354 return '-f -s in out array dhist'.split()
@@ -1,536 +1,536 b''
1 1 """DEPRECATED: Input transformer classes to support IPython special syntax.
2 2
3 3 This module was deprecated in IPython 7.0, in favour of inputtransformer2.
4 4
5 5 This includes the machinery to recognise and transform ``%magic`` commands,
6 6 ``!system`` commands, ``help?`` querying, prompt stripping, and so forth.
7 7 """
8 8 import abc
9 9 import functools
10 10 import re
11 11 import tokenize
12 12 from tokenize import generate_tokens, untokenize, TokenError
13 13 from io import StringIO
14 14
15 15 from IPython.core.splitinput import LineInfo
16 16
17 17 #-----------------------------------------------------------------------------
18 18 # Globals
19 19 #-----------------------------------------------------------------------------
20 20
21 21 # The escape sequences that define the syntax transformations IPython will
22 22 # apply to user input. These can NOT be just changed here: many regular
23 23 # expressions and other parts of the code may use their hardcoded values, and
24 24 # for all intents and purposes they constitute the 'IPython syntax', so they
25 25 # should be considered fixed.
26 26
27 27 ESC_SHELL = '!' # Send line to underlying system shell
28 28 ESC_SH_CAP = '!!' # Send line to system shell and capture output
29 29 ESC_HELP = '?' # Find information about object
30 30 ESC_HELP2 = '??' # Find extra-detailed information about object
31 31 ESC_MAGIC = '%' # Call magic function
32 32 ESC_MAGIC2 = '%%' # Call cell-magic function
33 33 ESC_QUOTE = ',' # Split args on whitespace, quote each as string and call
34 34 ESC_QUOTE2 = ';' # Quote all args as a single string, call
35 35 ESC_PAREN = '/' # Call first argument with rest of line as arguments
36 36
37 37 ESC_SEQUENCES = [ESC_SHELL, ESC_SH_CAP, ESC_HELP ,\
38 38 ESC_HELP2, ESC_MAGIC, ESC_MAGIC2,\
39 39 ESC_QUOTE, ESC_QUOTE2, ESC_PAREN ]
40 40
41 41
42 42 class InputTransformer(metaclass=abc.ABCMeta):
43 43 """Abstract base class for line-based input transformers."""
44 44
45 45 @abc.abstractmethod
46 46 def push(self, line):
47 47 """Send a line of input to the transformer, returning the transformed
48 48 input or None if the transformer is waiting for more input.
49 49
50 50 Must be overridden by subclasses.
51 51
52 52 Implementations may raise ``SyntaxError`` if the input is invalid. No
53 53 other exceptions may be raised.
54 54 """
55 55 pass
56 56
57 57 @abc.abstractmethod
58 58 def reset(self):
59 59 """Return, transformed any lines that the transformer has accumulated,
60 60 and reset its internal state.
61 61
62 62 Must be overridden by subclasses.
63 63 """
64 64 pass
65 65
66 66 @classmethod
67 67 def wrap(cls, func):
68 68 """Can be used by subclasses as a decorator, to return a factory that
69 69 will allow instantiation with the decorated object.
70 70 """
71 71 @functools.wraps(func)
72 72 def transformer_factory(**kwargs):
73 73 return cls(func, **kwargs)
74 74
75 75 return transformer_factory
76 76
77 77 class StatelessInputTransformer(InputTransformer):
78 78 """Wrapper for a stateless input transformer implemented as a function."""
79 79 def __init__(self, func):
80 80 self.func = func
81 81
82 82 def __repr__(self):
83 83 return "StatelessInputTransformer(func={0!r})".format(self.func)
84 84
85 85 def push(self, line):
86 86 """Send a line of input to the transformer, returning the
87 87 transformed input."""
88 88 return self.func(line)
89 89
90 90 def reset(self):
91 91 """No-op - exists for compatibility."""
92 92 pass
93 93
94 94 class CoroutineInputTransformer(InputTransformer):
95 95 """Wrapper for an input transformer implemented as a coroutine."""
96 96 def __init__(self, coro, **kwargs):
97 97 # Prime it
98 98 self.coro = coro(**kwargs)
99 99 next(self.coro)
100 100
101 101 def __repr__(self):
102 102 return "CoroutineInputTransformer(coro={0!r})".format(self.coro)
103 103
104 104 def push(self, line):
105 105 """Send a line of input to the transformer, returning the
106 106 transformed input or None if the transformer is waiting for more
107 107 input.
108 108 """
109 109 return self.coro.send(line)
110 110
111 111 def reset(self):
112 112 """Return, transformed any lines that the transformer has
113 113 accumulated, and reset its internal state.
114 114 """
115 115 return self.coro.send(None)
116 116
117 117 class TokenInputTransformer(InputTransformer):
118 118 """Wrapper for a token-based input transformer.
119 119
120 120 func should accept a list of tokens (5-tuples, see tokenize docs), and
121 121 return an iterable which can be passed to tokenize.untokenize().
122 122 """
123 123 def __init__(self, func):
124 124 self.func = func
125 125 self.buf = []
126 126 self.reset_tokenizer()
127 127
128 128 def reset_tokenizer(self):
129 129 it = iter(self.buf)
130 130 self.tokenizer = generate_tokens(it.__next__)
131 131
132 132 def push(self, line):
133 133 self.buf.append(line + '\n')
134 134 if all(l.isspace() for l in self.buf):
135 135 return self.reset()
136 136
137 137 tokens = []
138 138 stop_at_NL = False
139 139 try:
140 140 for intok in self.tokenizer:
141 141 tokens.append(intok)
142 142 t = intok[0]
143 143 if t == tokenize.NEWLINE or (stop_at_NL and t == tokenize.NL):
144 144 # Stop before we try to pull a line we don't have yet
145 145 break
146 146 elif t == tokenize.ERRORTOKEN:
147 147 stop_at_NL = True
148 148 except TokenError:
149 149 # Multi-line statement - stop and try again with the next line
150 150 self.reset_tokenizer()
151 151 return None
152 152
153 153 return self.output(tokens)
154 154
155 155 def output(self, tokens):
156 156 self.buf.clear()
157 157 self.reset_tokenizer()
158 158 return untokenize(self.func(tokens)).rstrip('\n')
159 159
160 160 def reset(self):
161 161 l = ''.join(self.buf)
162 162 self.buf.clear()
163 163 self.reset_tokenizer()
164 164 if l:
165 165 return l.rstrip('\n')
166 166
167 167 class assemble_python_lines(TokenInputTransformer):
168 168 def __init__(self):
169 169 super(assemble_python_lines, self).__init__(None)
170 170
171 171 def output(self, tokens):
172 172 return self.reset()
173 173
174 174 @CoroutineInputTransformer.wrap
175 175 def assemble_logical_lines():
176 176 r"""Join lines following explicit line continuations (\)"""
177 177 line = ''
178 178 while True:
179 179 line = (yield line)
180 180 if not line or line.isspace():
181 181 continue
182 182
183 183 parts = []
184 184 while line is not None:
185 185 if line.endswith('\\') and (not has_comment(line)):
186 186 parts.append(line[:-1])
187 187 line = (yield None) # Get another line
188 188 else:
189 189 parts.append(line)
190 190 break
191 191
192 192 # Output
193 193 line = ''.join(parts)
194 194
195 195 # Utilities
196 196 def _make_help_call(target, esc, lspace, next_input=None):
197 197 """Prepares a pinfo(2)/psearch call from a target name and the escape
198 198 (i.e. ? or ??)"""
199 199 method = 'pinfo2' if esc == '??' \
200 200 else 'psearch' if '*' in target \
201 201 else 'pinfo'
202 202 arg = " ".join([method, target])
203 203 #Prepare arguments for get_ipython().run_line_magic(magic_name, magic_args)
204 204 t_magic_name, _, t_magic_arg_s = arg.partition(' ')
205 205 t_magic_name = t_magic_name.lstrip(ESC_MAGIC)
206 206 if next_input is None:
207 207 return '%sget_ipython().run_line_magic(%r, %r)' % (lspace, t_magic_name, t_magic_arg_s)
208 208 else:
209 209 return '%sget_ipython().set_next_input(%r);get_ipython().run_line_magic(%r, %r)' % \
210 210 (lspace, next_input, t_magic_name, t_magic_arg_s)
211 211
212 212 # These define the transformations for the different escape characters.
213 213 def _tr_system(line_info):
214 214 "Translate lines escaped with: !"
215 215 cmd = line_info.line.lstrip().lstrip(ESC_SHELL)
216 216 return '%sget_ipython().system(%r)' % (line_info.pre, cmd)
217 217
218 218 def _tr_system2(line_info):
219 219 "Translate lines escaped with: !!"
220 220 cmd = line_info.line.lstrip()[2:]
221 221 return '%sget_ipython().getoutput(%r)' % (line_info.pre, cmd)
222 222
223 223 def _tr_help(line_info):
224 224 "Translate lines escaped with: ?/??"
225 225 # A naked help line should just fire the intro help screen
226 226 if not line_info.line[1:]:
227 227 return 'get_ipython().show_usage()'
228 228
229 229 return _make_help_call(line_info.ifun, line_info.esc, line_info.pre)
230 230
231 231 def _tr_magic(line_info):
232 232 "Translate lines escaped with: %"
233 233 tpl = '%sget_ipython().run_line_magic(%r, %r)'
234 234 if line_info.line.startswith(ESC_MAGIC2):
235 235 return line_info.line
236 236 cmd = ' '.join([line_info.ifun, line_info.the_rest]).strip()
237 237 #Prepare arguments for get_ipython().run_line_magic(magic_name, magic_args)
238 238 t_magic_name, _, t_magic_arg_s = cmd.partition(' ')
239 239 t_magic_name = t_magic_name.lstrip(ESC_MAGIC)
240 240 return tpl % (line_info.pre, t_magic_name, t_magic_arg_s)
241 241
242 242 def _tr_quote(line_info):
243 243 "Translate lines escaped with: ,"
244 244 return '%s%s("%s")' % (line_info.pre, line_info.ifun,
245 245 '", "'.join(line_info.the_rest.split()) )
246 246
247 247 def _tr_quote2(line_info):
248 248 "Translate lines escaped with: ;"
249 249 return '%s%s("%s")' % (line_info.pre, line_info.ifun,
250 250 line_info.the_rest)
251 251
252 252 def _tr_paren(line_info):
253 253 "Translate lines escaped with: /"
254 254 return '%s%s(%s)' % (line_info.pre, line_info.ifun,
255 255 ", ".join(line_info.the_rest.split()))
256 256
257 257 tr = { ESC_SHELL : _tr_system,
258 258 ESC_SH_CAP : _tr_system2,
259 259 ESC_HELP : _tr_help,
260 260 ESC_HELP2 : _tr_help,
261 261 ESC_MAGIC : _tr_magic,
262 262 ESC_QUOTE : _tr_quote,
263 263 ESC_QUOTE2 : _tr_quote2,
264 264 ESC_PAREN : _tr_paren }
265 265
266 266 @StatelessInputTransformer.wrap
267 267 def escaped_commands(line):
268 268 """Transform escaped commands - %magic, !system, ?help + various autocalls.
269 269 """
270 270 if not line or line.isspace():
271 271 return line
272 272 lineinf = LineInfo(line)
273 273 if lineinf.esc not in tr:
274 274 return line
275 275
276 276 return tr[lineinf.esc](lineinf)
277 277
278 278 _initial_space_re = re.compile(r'\s*')
279 279
280 280 _help_end_re = re.compile(r"""(%{0,2}
281 [a-zA-Z_*][\w*]* # Variable name
282 (\.[a-zA-Z_*][\w*]*)* # .etc.etc
281 (?!\d)[\w*]+ # Variable name
282 (\.(?!\d)[\w*]+)* # .etc.etc
283 283 )
284 284 (\?\??)$ # ? or ??
285 285 """,
286 286 re.VERBOSE)
287 287
288 288 # Extra pseudotokens for multiline strings and data structures
289 289 _MULTILINE_STRING = object()
290 290 _MULTILINE_STRUCTURE = object()
291 291
292 292 def _line_tokens(line):
293 293 """Helper for has_comment and ends_in_comment_or_string."""
294 294 readline = StringIO(line).readline
295 295 toktypes = set()
296 296 try:
297 297 for t in generate_tokens(readline):
298 298 toktypes.add(t[0])
299 299 except TokenError as e:
300 300 # There are only two cases where a TokenError is raised.
301 301 if 'multi-line string' in e.args[0]:
302 302 toktypes.add(_MULTILINE_STRING)
303 303 else:
304 304 toktypes.add(_MULTILINE_STRUCTURE)
305 305 return toktypes
306 306
307 307 def has_comment(src):
308 308 """Indicate whether an input line has (i.e. ends in, or is) a comment.
309 309
310 310 This uses tokenize, so it can distinguish comments from # inside strings.
311 311
312 312 Parameters
313 313 ----------
314 314 src : string
315 315 A single line input string.
316 316
317 317 Returns
318 318 -------
319 319 comment : bool
320 320 True if source has a comment.
321 321 """
322 322 return (tokenize.COMMENT in _line_tokens(src))
323 323
324 324 def ends_in_comment_or_string(src):
325 325 """Indicates whether or not an input line ends in a comment or within
326 326 a multiline string.
327 327
328 328 Parameters
329 329 ----------
330 330 src : string
331 331 A single line input string.
332 332
333 333 Returns
334 334 -------
335 335 comment : bool
336 336 True if source ends in a comment or multiline string.
337 337 """
338 338 toktypes = _line_tokens(src)
339 339 return (tokenize.COMMENT in toktypes) or (_MULTILINE_STRING in toktypes)
340 340
341 341
342 342 @StatelessInputTransformer.wrap
343 343 def help_end(line):
344 344 """Translate lines with ?/?? at the end"""
345 345 m = _help_end_re.search(line)
346 346 if m is None or ends_in_comment_or_string(line):
347 347 return line
348 348 target = m.group(1)
349 349 esc = m.group(3)
350 350 lspace = _initial_space_re.match(line).group(0)
351 351
352 352 # If we're mid-command, put it back on the next prompt for the user.
353 353 next_input = line.rstrip('?') if line.strip() != m.group(0) else None
354 354
355 355 return _make_help_call(target, esc, lspace, next_input)
356 356
357 357
358 358 @CoroutineInputTransformer.wrap
359 359 def cellmagic(end_on_blank_line=False):
360 360 """Captures & transforms cell magics.
361 361
362 362 After a cell magic is started, this stores up any lines it gets until it is
363 363 reset (sent None).
364 364 """
365 365 tpl = 'get_ipython().run_cell_magic(%r, %r, %r)'
366 366 cellmagic_help_re = re.compile(r'%%\w+\?')
367 367 line = ''
368 368 while True:
369 369 line = (yield line)
370 370 # consume leading empty lines
371 371 while not line:
372 372 line = (yield line)
373 373
374 374 if not line.startswith(ESC_MAGIC2):
375 375 # This isn't a cell magic, idle waiting for reset then start over
376 376 while line is not None:
377 377 line = (yield line)
378 378 continue
379 379
380 380 if cellmagic_help_re.match(line):
381 381 # This case will be handled by help_end
382 382 continue
383 383
384 384 first = line
385 385 body = []
386 386 line = (yield None)
387 387 while (line is not None) and \
388 388 ((line.strip() != '') or not end_on_blank_line):
389 389 body.append(line)
390 390 line = (yield None)
391 391
392 392 # Output
393 393 magic_name, _, first = first.partition(' ')
394 394 magic_name = magic_name.lstrip(ESC_MAGIC2)
395 395 line = tpl % (magic_name, first, u'\n'.join(body))
396 396
397 397
398 398 def _strip_prompts(prompt_re, initial_re=None, turnoff_re=None):
399 399 """Remove matching input prompts from a block of input.
400 400
401 401 Parameters
402 402 ----------
403 403 prompt_re : regular expression
404 404 A regular expression matching any input prompt (including continuation)
405 405 initial_re : regular expression, optional
406 406 A regular expression matching only the initial prompt, but not continuation.
407 407 If no initial expression is given, prompt_re will be used everywhere.
408 408 Used mainly for plain Python prompts, where the continuation prompt
409 409 ``...`` is a valid Python expression in Python 3, so shouldn't be stripped.
410 410
411 411 If initial_re and prompt_re differ,
412 412 only initial_re will be tested against the first line.
413 413 If any prompt is found on the first two lines,
414 414 prompts will be stripped from the rest of the block.
415 415 """
416 416 if initial_re is None:
417 417 initial_re = prompt_re
418 418 line = ''
419 419 while True:
420 420 line = (yield line)
421 421
422 422 # First line of cell
423 423 if line is None:
424 424 continue
425 425 out, n1 = initial_re.subn('', line, count=1)
426 426 if turnoff_re and not n1:
427 427 if turnoff_re.match(line):
428 428 # We're in e.g. a cell magic; disable this transformer for
429 429 # the rest of the cell.
430 430 while line is not None:
431 431 line = (yield line)
432 432 continue
433 433
434 434 line = (yield out)
435 435
436 436 if line is None:
437 437 continue
438 438 # check for any prompt on the second line of the cell,
439 439 # because people often copy from just after the first prompt,
440 440 # so we might not see it in the first line.
441 441 out, n2 = prompt_re.subn('', line, count=1)
442 442 line = (yield out)
443 443
444 444 if n1 or n2:
445 445 # Found a prompt in the first two lines - check for it in
446 446 # the rest of the cell as well.
447 447 while line is not None:
448 448 line = (yield prompt_re.sub('', line, count=1))
449 449
450 450 else:
451 451 # Prompts not in input - wait for reset
452 452 while line is not None:
453 453 line = (yield line)
454 454
455 455 @CoroutineInputTransformer.wrap
456 456 def classic_prompt():
457 457 """Strip the >>>/... prompts of the Python interactive shell."""
458 458 # FIXME: non-capturing version (?:...) usable?
459 459 prompt_re = re.compile(r'^(>>>|\.\.\.)( |$)')
460 460 initial_re = re.compile(r'^>>>( |$)')
461 461 # Any %magic/!system is IPython syntax, so we needn't look for >>> prompts
462 462 turnoff_re = re.compile(r'^[%!]')
463 463 return _strip_prompts(prompt_re, initial_re, turnoff_re)
464 464
465 465 @CoroutineInputTransformer.wrap
466 466 def ipy_prompt():
467 467 """Strip IPython's In [1]:/...: prompts."""
468 468 # FIXME: non-capturing version (?:...) usable?
469 469 prompt_re = re.compile(r'^(In \[\d+\]: |\s*\.{3,}: ?)')
470 470 # Disable prompt stripping inside cell magics
471 471 turnoff_re = re.compile(r'^%%')
472 472 return _strip_prompts(prompt_re, turnoff_re=turnoff_re)
473 473
474 474
475 475 @CoroutineInputTransformer.wrap
476 476 def leading_indent():
477 477 """Remove leading indentation.
478 478
479 479 If the first line starts with a spaces or tabs, the same whitespace will be
480 480 removed from each following line until it is reset.
481 481 """
482 482 space_re = re.compile(r'^[ \t]+')
483 483 line = ''
484 484 while True:
485 485 line = (yield line)
486 486
487 487 if line is None:
488 488 continue
489 489
490 490 m = space_re.match(line)
491 491 if m:
492 492 space = m.group(0)
493 493 while line is not None:
494 494 if line.startswith(space):
495 495 line = line[len(space):]
496 496 line = (yield line)
497 497 else:
498 498 # No leading spaces - wait for reset
499 499 while line is not None:
500 500 line = (yield line)
501 501
502 502
503 503 _assign_pat = \
504 504 r'''(?P<lhs>(\s*)
505 505 ([\w\.]+) # Initial identifier
506 506 (\s*,\s*
507 507 \*?[\w\.]+)* # Further identifiers for unpacking
508 508 \s*?,? # Trailing comma
509 509 )
510 510 \s*=\s*
511 511 '''
512 512
513 513 assign_system_re = re.compile(r'{}!\s*(?P<cmd>.*)'.format(_assign_pat), re.VERBOSE)
514 514 assign_system_template = '%s = get_ipython().getoutput(%r)'
515 515 @StatelessInputTransformer.wrap
516 516 def assign_from_system(line):
517 517 """Transform assignment from system commands (e.g. files = !ls)"""
518 518 m = assign_system_re.match(line)
519 519 if m is None:
520 520 return line
521 521
522 522 return assign_system_template % m.group('lhs', 'cmd')
523 523
524 524 assign_magic_re = re.compile(r'{}%\s*(?P<cmd>.*)'.format(_assign_pat), re.VERBOSE)
525 525 assign_magic_template = '%s = get_ipython().run_line_magic(%r, %r)'
526 526 @StatelessInputTransformer.wrap
527 527 def assign_from_magic(line):
528 528 """Transform assignment from magic commands (e.g. a = %who_ls)"""
529 529 m = assign_magic_re.match(line)
530 530 if m is None:
531 531 return line
532 532 #Prepare arguments for get_ipython().run_line_magic(magic_name, magic_args)
533 533 m_lhs, m_cmd = m.group('lhs', 'cmd')
534 534 t_magic_name, _, t_magic_arg_s = m_cmd.partition(' ')
535 535 t_magic_name = t_magic_name.lstrip(ESC_MAGIC)
536 536 return assign_magic_template % (m_lhs, t_magic_name, t_magic_arg_s)
@@ -1,721 +1,721 b''
1 1 """Input transformer machinery to support IPython special syntax.
2 2
3 3 This includes the machinery to recognise and transform ``%magic`` commands,
4 4 ``!system`` commands, ``help?`` querying, prompt stripping, and so forth.
5 5
6 6 Added: IPython 7.0. Replaces inputsplitter and inputtransformer which were
7 7 deprecated in 7.0.
8 8 """
9 9
10 10 # Copyright (c) IPython Development Team.
11 11 # Distributed under the terms of the Modified BSD License.
12 12
13 13 from codeop import compile_command
14 14 import re
15 15 import tokenize
16 16 from typing import List, Tuple, Union
17 17 import warnings
18 18
19 19 _indent_re = re.compile(r'^[ \t]+')
20 20
21 21 def leading_empty_lines(lines):
22 22 """Remove leading empty lines
23 23
24 24 If the leading lines are empty or contain only whitespace, they will be
25 25 removed.
26 26 """
27 27 if not lines:
28 28 return lines
29 29 for i, line in enumerate(lines):
30 30 if line and not line.isspace():
31 31 return lines[i:]
32 32 return lines
33 33
34 34 def leading_indent(lines):
35 35 """Remove leading indentation.
36 36
37 37 If the first line starts with a spaces or tabs, the same whitespace will be
38 38 removed from each following line in the cell.
39 39 """
40 40 if not lines:
41 41 return lines
42 42 m = _indent_re.match(lines[0])
43 43 if not m:
44 44 return lines
45 45 space = m.group(0)
46 46 n = len(space)
47 47 return [l[n:] if l.startswith(space) else l
48 48 for l in lines]
49 49
50 50 class PromptStripper:
51 51 """Remove matching input prompts from a block of input.
52 52
53 53 Parameters
54 54 ----------
55 55 prompt_re : regular expression
56 56 A regular expression matching any input prompt (including continuation,
57 57 e.g. ``...``)
58 58 initial_re : regular expression, optional
59 59 A regular expression matching only the initial prompt, but not continuation.
60 60 If no initial expression is given, prompt_re will be used everywhere.
61 61 Used mainly for plain Python prompts (``>>>``), where the continuation prompt
62 62 ``...`` is a valid Python expression in Python 3, so shouldn't be stripped.
63 63
64 64 If initial_re and prompt_re differ,
65 65 only initial_re will be tested against the first line.
66 66 If any prompt is found on the first two lines,
67 67 prompts will be stripped from the rest of the block.
68 68 """
69 69 def __init__(self, prompt_re, initial_re=None):
70 70 self.prompt_re = prompt_re
71 71 self.initial_re = initial_re or prompt_re
72 72
73 73 def _strip(self, lines):
74 74 return [self.prompt_re.sub('', l, count=1) for l in lines]
75 75
76 76 def __call__(self, lines):
77 77 if not lines:
78 78 return lines
79 79 if self.initial_re.match(lines[0]) or \
80 80 (len(lines) > 1 and self.prompt_re.match(lines[1])):
81 81 return self._strip(lines)
82 82 return lines
83 83
84 84 classic_prompt = PromptStripper(
85 85 prompt_re=re.compile(r'^(>>>|\.\.\.)( |$)'),
86 86 initial_re=re.compile(r'^>>>( |$)')
87 87 )
88 88
89 89 ipython_prompt = PromptStripper(re.compile(r'^(In \[\d+\]: |\s*\.{3,}: ?)'))
90 90
91 91 def cell_magic(lines):
92 92 if not lines or not lines[0].startswith('%%'):
93 93 return lines
94 94 if re.match(r'%%\w+\?', lines[0]):
95 95 # This case will be handled by help_end
96 96 return lines
97 97 magic_name, _, first_line = lines[0][2:-1].partition(' ')
98 98 body = ''.join(lines[1:])
99 99 return ['get_ipython().run_cell_magic(%r, %r, %r)\n'
100 100 % (magic_name, first_line, body)]
101 101
102 102
103 103 def _find_assign_op(token_line) -> Union[int, None]:
104 104 """Get the index of the first assignment in the line ('=' not inside brackets)
105 105
106 106 Note: We don't try to support multiple special assignment (a = b = %foo)
107 107 """
108 108 paren_level = 0
109 109 for i, ti in enumerate(token_line):
110 110 s = ti.string
111 111 if s == '=' and paren_level == 0:
112 112 return i
113 113 if s in {'(','[','{'}:
114 114 paren_level += 1
115 115 elif s in {')', ']', '}'}:
116 116 if paren_level > 0:
117 117 paren_level -= 1
118 118
119 119 def find_end_of_continued_line(lines, start_line: int):
120 120 """Find the last line of a line explicitly extended using backslashes.
121 121
122 122 Uses 0-indexed line numbers.
123 123 """
124 124 end_line = start_line
125 125 while lines[end_line].endswith('\\\n'):
126 126 end_line += 1
127 127 if end_line >= len(lines):
128 128 break
129 129 return end_line
130 130
131 131 def assemble_continued_line(lines, start: Tuple[int, int], end_line: int):
132 132 r"""Assemble a single line from multiple continued line pieces
133 133
134 134 Continued lines are lines ending in ``\``, and the line following the last
135 135 ``\`` in the block.
136 136
137 137 For example, this code continues over multiple lines::
138 138
139 139 if (assign_ix is not None) \
140 140 and (len(line) >= assign_ix + 2) \
141 141 and (line[assign_ix+1].string == '%') \
142 142 and (line[assign_ix+2].type == tokenize.NAME):
143 143
144 144 This statement contains four continued line pieces.
145 145 Assembling these pieces into a single line would give::
146 146
147 147 if (assign_ix is not None) and (len(line) >= assign_ix + 2) and (line[...
148 148
149 149 This uses 0-indexed line numbers. *start* is (lineno, colno).
150 150
151 151 Used to allow ``%magic`` and ``!system`` commands to be continued over
152 152 multiple lines.
153 153 """
154 154 parts = [lines[start[0]][start[1]:]] + lines[start[0]+1:end_line+1]
155 155 return ' '.join([p[:-2] for p in parts[:-1]] # Strip backslash+newline
156 156 + [parts[-1][:-1]]) # Strip newline from last line
157 157
158 158 class TokenTransformBase:
159 159 """Base class for transformations which examine tokens.
160 160
161 161 Special syntax should not be transformed when it occurs inside strings or
162 162 comments. This is hard to reliably avoid with regexes. The solution is to
163 163 tokenise the code as Python, and recognise the special syntax in the tokens.
164 164
165 165 IPython's special syntax is not valid Python syntax, so tokenising may go
166 166 wrong after the special syntax starts. These classes therefore find and
167 167 transform *one* instance of special syntax at a time into regular Python
168 168 syntax. After each transformation, tokens are regenerated to find the next
169 169 piece of special syntax.
170 170
171 171 Subclasses need to implement one class method (find)
172 172 and one regular method (transform).
173 173
174 174 The priority attribute can select which transformation to apply if multiple
175 175 transformers match in the same place. Lower numbers have higher priority.
176 176 This allows "%magic?" to be turned into a help call rather than a magic call.
177 177 """
178 178 # Lower numbers -> higher priority (for matches in the same location)
179 179 priority = 10
180 180
181 181 def sortby(self):
182 182 return self.start_line, self.start_col, self.priority
183 183
184 184 def __init__(self, start):
185 185 self.start_line = start[0] - 1 # Shift from 1-index to 0-index
186 186 self.start_col = start[1]
187 187
188 188 @classmethod
189 189 def find(cls, tokens_by_line):
190 190 """Find one instance of special syntax in the provided tokens.
191 191
192 192 Tokens are grouped into logical lines for convenience,
193 193 so it is easy to e.g. look at the first token of each line.
194 194 *tokens_by_line* is a list of lists of tokenize.TokenInfo objects.
195 195
196 196 This should return an instance of its class, pointing to the start
197 197 position it has found, or None if it found no match.
198 198 """
199 199 raise NotImplementedError
200 200
201 201 def transform(self, lines: List[str]):
202 202 """Transform one instance of special syntax found by ``find()``
203 203
204 204 Takes a list of strings representing physical lines,
205 205 returns a similar list of transformed lines.
206 206 """
207 207 raise NotImplementedError
208 208
209 209 class MagicAssign(TokenTransformBase):
210 210 """Transformer for assignments from magics (a = %foo)"""
211 211 @classmethod
212 212 def find(cls, tokens_by_line):
213 213 """Find the first magic assignment (a = %foo) in the cell.
214 214 """
215 215 for line in tokens_by_line:
216 216 assign_ix = _find_assign_op(line)
217 217 if (assign_ix is not None) \
218 218 and (len(line) >= assign_ix + 2) \
219 219 and (line[assign_ix+1].string == '%') \
220 220 and (line[assign_ix+2].type == tokenize.NAME):
221 221 return cls(line[assign_ix+1].start)
222 222
223 223 def transform(self, lines: List[str]):
224 224 """Transform a magic assignment found by the ``find()`` classmethod.
225 225 """
226 226 start_line, start_col = self.start_line, self.start_col
227 227 lhs = lines[start_line][:start_col]
228 228 end_line = find_end_of_continued_line(lines, start_line)
229 229 rhs = assemble_continued_line(lines, (start_line, start_col), end_line)
230 230 assert rhs.startswith('%'), rhs
231 231 magic_name, _, args = rhs[1:].partition(' ')
232 232
233 233 lines_before = lines[:start_line]
234 234 call = "get_ipython().run_line_magic({!r}, {!r})".format(magic_name, args)
235 235 new_line = lhs + call + '\n'
236 236 lines_after = lines[end_line+1:]
237 237
238 238 return lines_before + [new_line] + lines_after
239 239
240 240
241 241 class SystemAssign(TokenTransformBase):
242 242 """Transformer for assignments from system commands (a = !foo)"""
243 243 @classmethod
244 244 def find(cls, tokens_by_line):
245 245 """Find the first system assignment (a = !foo) in the cell.
246 246 """
247 247 for line in tokens_by_line:
248 248 assign_ix = _find_assign_op(line)
249 249 if (assign_ix is not None) \
250 250 and not line[assign_ix].line.strip().startswith('=') \
251 251 and (len(line) >= assign_ix + 2) \
252 252 and (line[assign_ix + 1].type == tokenize.ERRORTOKEN):
253 253 ix = assign_ix + 1
254 254
255 255 while ix < len(line) and line[ix].type == tokenize.ERRORTOKEN:
256 256 if line[ix].string == '!':
257 257 return cls(line[ix].start)
258 258 elif not line[ix].string.isspace():
259 259 break
260 260 ix += 1
261 261
262 262 def transform(self, lines: List[str]):
263 263 """Transform a system assignment found by the ``find()`` classmethod.
264 264 """
265 265 start_line, start_col = self.start_line, self.start_col
266 266
267 267 lhs = lines[start_line][:start_col]
268 268 end_line = find_end_of_continued_line(lines, start_line)
269 269 rhs = assemble_continued_line(lines, (start_line, start_col), end_line)
270 270 assert rhs.startswith('!'), rhs
271 271 cmd = rhs[1:]
272 272
273 273 lines_before = lines[:start_line]
274 274 call = "get_ipython().getoutput({!r})".format(cmd)
275 275 new_line = lhs + call + '\n'
276 276 lines_after = lines[end_line + 1:]
277 277
278 278 return lines_before + [new_line] + lines_after
279 279
280 280 # The escape sequences that define the syntax transformations IPython will
281 281 # apply to user input. These can NOT be just changed here: many regular
282 282 # expressions and other parts of the code may use their hardcoded values, and
283 283 # for all intents and purposes they constitute the 'IPython syntax', so they
284 284 # should be considered fixed.
285 285
286 286 ESC_SHELL = '!' # Send line to underlying system shell
287 287 ESC_SH_CAP = '!!' # Send line to system shell and capture output
288 288 ESC_HELP = '?' # Find information about object
289 289 ESC_HELP2 = '??' # Find extra-detailed information about object
290 290 ESC_MAGIC = '%' # Call magic function
291 291 ESC_MAGIC2 = '%%' # Call cell-magic function
292 292 ESC_QUOTE = ',' # Split args on whitespace, quote each as string and call
293 293 ESC_QUOTE2 = ';' # Quote all args as a single string, call
294 294 ESC_PAREN = '/' # Call first argument with rest of line as arguments
295 295
296 296 ESCAPE_SINGLES = {'!', '?', '%', ',', ';', '/'}
297 297 ESCAPE_DOUBLES = {'!!', '??'} # %% (cell magic) is handled separately
298 298
299 299 def _make_help_call(target, esc, next_input=None):
300 300 """Prepares a pinfo(2)/psearch call from a target name and the escape
301 301 (i.e. ? or ??)"""
302 302 method = 'pinfo2' if esc == '??' \
303 303 else 'psearch' if '*' in target \
304 304 else 'pinfo'
305 305 arg = " ".join([method, target])
306 306 #Prepare arguments for get_ipython().run_line_magic(magic_name, magic_args)
307 307 t_magic_name, _, t_magic_arg_s = arg.partition(' ')
308 308 t_magic_name = t_magic_name.lstrip(ESC_MAGIC)
309 309 if next_input is None:
310 310 return 'get_ipython().run_line_magic(%r, %r)' % (t_magic_name, t_magic_arg_s)
311 311 else:
312 312 return 'get_ipython().set_next_input(%r);get_ipython().run_line_magic(%r, %r)' % \
313 313 (next_input, t_magic_name, t_magic_arg_s)
314 314
315 315 def _tr_help(content):
316 316 """Translate lines escaped with: ?
317 317
318 318 A naked help line should fire the intro help screen (shell.show_usage())
319 319 """
320 320 if not content:
321 321 return 'get_ipython().show_usage()'
322 322
323 323 return _make_help_call(content, '?')
324 324
325 325 def _tr_help2(content):
326 326 """Translate lines escaped with: ??
327 327
328 328 A naked help line should fire the intro help screen (shell.show_usage())
329 329 """
330 330 if not content:
331 331 return 'get_ipython().show_usage()'
332 332
333 333 return _make_help_call(content, '??')
334 334
335 335 def _tr_magic(content):
336 336 "Translate lines escaped with a percent sign: %"
337 337 name, _, args = content.partition(' ')
338 338 return 'get_ipython().run_line_magic(%r, %r)' % (name, args)
339 339
340 340 def _tr_quote(content):
341 341 "Translate lines escaped with a comma: ,"
342 342 name, _, args = content.partition(' ')
343 343 return '%s("%s")' % (name, '", "'.join(args.split()) )
344 344
345 345 def _tr_quote2(content):
346 346 "Translate lines escaped with a semicolon: ;"
347 347 name, _, args = content.partition(' ')
348 348 return '%s("%s")' % (name, args)
349 349
350 350 def _tr_paren(content):
351 351 "Translate lines escaped with a slash: /"
352 352 name, _, args = content.partition(' ')
353 353 return '%s(%s)' % (name, ", ".join(args.split()))
354 354
355 355 tr = { ESC_SHELL : 'get_ipython().system({!r})'.format,
356 356 ESC_SH_CAP : 'get_ipython().getoutput({!r})'.format,
357 357 ESC_HELP : _tr_help,
358 358 ESC_HELP2 : _tr_help2,
359 359 ESC_MAGIC : _tr_magic,
360 360 ESC_QUOTE : _tr_quote,
361 361 ESC_QUOTE2 : _tr_quote2,
362 362 ESC_PAREN : _tr_paren }
363 363
364 364 class EscapedCommand(TokenTransformBase):
365 365 """Transformer for escaped commands like %foo, !foo, or /foo"""
366 366 @classmethod
367 367 def find(cls, tokens_by_line):
368 368 """Find the first escaped command (%foo, !foo, etc.) in the cell.
369 369 """
370 370 for line in tokens_by_line:
371 371 if not line:
372 372 continue
373 373 ix = 0
374 374 ll = len(line)
375 375 while ll > ix and line[ix].type in {tokenize.INDENT, tokenize.DEDENT}:
376 376 ix += 1
377 377 if ix >= ll:
378 378 continue
379 379 if line[ix].string in ESCAPE_SINGLES:
380 380 return cls(line[ix].start)
381 381
382 382 def transform(self, lines):
383 383 """Transform an escaped line found by the ``find()`` classmethod.
384 384 """
385 385 start_line, start_col = self.start_line, self.start_col
386 386
387 387 indent = lines[start_line][:start_col]
388 388 end_line = find_end_of_continued_line(lines, start_line)
389 389 line = assemble_continued_line(lines, (start_line, start_col), end_line)
390 390
391 391 if len(line) > 1 and line[:2] in ESCAPE_DOUBLES:
392 392 escape, content = line[:2], line[2:]
393 393 else:
394 394 escape, content = line[:1], line[1:]
395 395
396 396 if escape in tr:
397 397 call = tr[escape](content)
398 398 else:
399 399 call = ''
400 400
401 401 lines_before = lines[:start_line]
402 402 new_line = indent + call + '\n'
403 403 lines_after = lines[end_line + 1:]
404 404
405 405 return lines_before + [new_line] + lines_after
406 406
407 407 _help_end_re = re.compile(r"""(%{0,2}
408 [a-zA-Z_*][\w*]* # Variable name
409 (\.[a-zA-Z_*][\w*]*)* # .etc.etc
408 (?!\d)[\w*]+ # Variable name
409 (\.(?!\d)[\w*]+)* # .etc.etc
410 410 )
411 411 (\?\??)$ # ? or ??
412 412 """,
413 413 re.VERBOSE)
414 414
415 415 class HelpEnd(TokenTransformBase):
416 416 """Transformer for help syntax: obj? and obj??"""
417 417 # This needs to be higher priority (lower number) than EscapedCommand so
418 418 # that inspecting magics (%foo?) works.
419 419 priority = 5
420 420
421 421 def __init__(self, start, q_locn):
422 422 super().__init__(start)
423 423 self.q_line = q_locn[0] - 1 # Shift from 1-indexed to 0-indexed
424 424 self.q_col = q_locn[1]
425 425
426 426 @classmethod
427 427 def find(cls, tokens_by_line):
428 428 """Find the first help command (foo?) in the cell.
429 429 """
430 430 for line in tokens_by_line:
431 431 # Last token is NEWLINE; look at last but one
432 432 if len(line) > 2 and line[-2].string == '?':
433 433 # Find the first token that's not INDENT/DEDENT
434 434 ix = 0
435 435 while line[ix].type in {tokenize.INDENT, tokenize.DEDENT}:
436 436 ix += 1
437 437 return cls(line[ix].start, line[-2].start)
438 438
439 439 def transform(self, lines):
440 440 """Transform a help command found by the ``find()`` classmethod.
441 441 """
442 442 piece = ''.join(lines[self.start_line:self.q_line+1])
443 443 indent, content = piece[:self.start_col], piece[self.start_col:]
444 444 lines_before = lines[:self.start_line]
445 445 lines_after = lines[self.q_line + 1:]
446 446
447 447 m = _help_end_re.search(content)
448 448 if not m:
449 449 raise SyntaxError(content)
450 450 assert m is not None, content
451 451 target = m.group(1)
452 452 esc = m.group(3)
453 453
454 454 # If we're mid-command, put it back on the next prompt for the user.
455 455 next_input = None
456 456 if (not lines_before) and (not lines_after) \
457 457 and content.strip() != m.group(0):
458 458 next_input = content.rstrip('?\n')
459 459
460 460 call = _make_help_call(target, esc, next_input=next_input)
461 461 new_line = indent + call + '\n'
462 462
463 463 return lines_before + [new_line] + lines_after
464 464
465 465 def make_tokens_by_line(lines:List[str]):
466 466 """Tokenize a series of lines and group tokens by line.
467 467
468 468 The tokens for a multiline Python string or expression are grouped as one
469 469 line. All lines except the last lines should keep their line ending ('\\n',
470 470 '\\r\\n') for this to properly work. Use `.splitlines(keeplineending=True)`
471 471 for example when passing block of text to this function.
472 472
473 473 """
474 474 # NL tokens are used inside multiline expressions, but also after blank
475 475 # lines or comments. This is intentional - see https://bugs.python.org/issue17061
476 476 # We want to group the former case together but split the latter, so we
477 477 # track parentheses level, similar to the internals of tokenize.
478 478 NEWLINE, NL = tokenize.NEWLINE, tokenize.NL
479 479 tokens_by_line = [[]]
480 480 if len(lines) > 1 and not lines[0].endswith(('\n', '\r', '\r\n', '\x0b', '\x0c')):
481 481 warnings.warn("`make_tokens_by_line` received a list of lines which do not have lineending markers ('\\n', '\\r', '\\r\\n', '\\x0b', '\\x0c'), behavior will be unspecified")
482 482 parenlev = 0
483 483 try:
484 484 for token in tokenize.generate_tokens(iter(lines).__next__):
485 485 tokens_by_line[-1].append(token)
486 486 if (token.type == NEWLINE) \
487 487 or ((token.type == NL) and (parenlev <= 0)):
488 488 tokens_by_line.append([])
489 489 elif token.string in {'(', '[', '{'}:
490 490 parenlev += 1
491 491 elif token.string in {')', ']', '}'}:
492 492 if parenlev > 0:
493 493 parenlev -= 1
494 494 except tokenize.TokenError:
495 495 # Input ended in a multiline string or expression. That's OK for us.
496 496 pass
497 497
498 498
499 499 if not tokens_by_line[-1]:
500 500 tokens_by_line.pop()
501 501
502 502
503 503 return tokens_by_line
504 504
505 505 def show_linewise_tokens(s: str):
506 506 """For investigation and debugging"""
507 507 if not s.endswith('\n'):
508 508 s += '\n'
509 509 lines = s.splitlines(keepends=True)
510 510 for line in make_tokens_by_line(lines):
511 511 print("Line -------")
512 512 for tokinfo in line:
513 513 print(" ", tokinfo)
514 514
515 515 # Arbitrary limit to prevent getting stuck in infinite loops
516 516 TRANSFORM_LOOP_LIMIT = 500
517 517
518 518 class TransformerManager:
519 519 """Applies various transformations to a cell or code block.
520 520
521 521 The key methods for external use are ``transform_cell()``
522 522 and ``check_complete()``.
523 523 """
524 524 def __init__(self):
525 525 self.cleanup_transforms = [
526 526 leading_empty_lines,
527 527 leading_indent,
528 528 classic_prompt,
529 529 ipython_prompt,
530 530 ]
531 531 self.line_transforms = [
532 532 cell_magic,
533 533 ]
534 534 self.token_transformers = [
535 535 MagicAssign,
536 536 SystemAssign,
537 537 EscapedCommand,
538 538 HelpEnd,
539 539 ]
540 540
541 541 def do_one_token_transform(self, lines):
542 542 """Find and run the transform earliest in the code.
543 543
544 544 Returns (changed, lines).
545 545
546 546 This method is called repeatedly until changed is False, indicating
547 547 that all available transformations are complete.
548 548
549 549 The tokens following IPython special syntax might not be valid, so
550 550 the transformed code is retokenised every time to identify the next
551 551 piece of special syntax. Hopefully long code cells are mostly valid
552 552 Python, not using lots of IPython special syntax, so this shouldn't be
553 553 a performance issue.
554 554 """
555 555 tokens_by_line = make_tokens_by_line(lines)
556 556 candidates = []
557 557 for transformer_cls in self.token_transformers:
558 558 transformer = transformer_cls.find(tokens_by_line)
559 559 if transformer:
560 560 candidates.append(transformer)
561 561
562 562 if not candidates:
563 563 # Nothing to transform
564 564 return False, lines
565 565 ordered_transformers = sorted(candidates, key=TokenTransformBase.sortby)
566 566 for transformer in ordered_transformers:
567 567 try:
568 568 return True, transformer.transform(lines)
569 569 except SyntaxError:
570 570 pass
571 571 return False, lines
572 572
573 573 def do_token_transforms(self, lines):
574 574 for _ in range(TRANSFORM_LOOP_LIMIT):
575 575 changed, lines = self.do_one_token_transform(lines)
576 576 if not changed:
577 577 return lines
578 578
579 579 raise RuntimeError("Input transformation still changing after "
580 580 "%d iterations. Aborting." % TRANSFORM_LOOP_LIMIT)
581 581
582 582 def transform_cell(self, cell: str) -> str:
583 583 """Transforms a cell of input code"""
584 584 if not cell.endswith('\n'):
585 585 cell += '\n' # Ensure the cell has a trailing newline
586 586 lines = cell.splitlines(keepends=True)
587 587 for transform in self.cleanup_transforms + self.line_transforms:
588 588 lines = transform(lines)
589 589
590 590 lines = self.do_token_transforms(lines)
591 591 return ''.join(lines)
592 592
593 593 def check_complete(self, cell: str):
594 594 """Return whether a block of code is ready to execute, or should be continued
595 595
596 596 Parameters
597 597 ----------
598 598 source : string
599 599 Python input code, which can be multiline.
600 600
601 601 Returns
602 602 -------
603 603 status : str
604 604 One of 'complete', 'incomplete', or 'invalid' if source is not a
605 605 prefix of valid code.
606 606 indent_spaces : int or None
607 607 The number of spaces by which to indent the next line of code. If
608 608 status is not 'incomplete', this is None.
609 609 """
610 610 # Remember if the lines ends in a new line.
611 611 ends_with_newline = False
612 612 for character in reversed(cell):
613 613 if character == '\n':
614 614 ends_with_newline = True
615 615 break
616 616 elif character.strip():
617 617 break
618 618 else:
619 619 continue
620 620
621 621 if not ends_with_newline:
622 622 # Append an newline for consistent tokenization
623 623 # See https://bugs.python.org/issue33899
624 624 cell += '\n'
625 625
626 626 lines = cell.splitlines(keepends=True)
627 627
628 628 if not lines:
629 629 return 'complete', None
630 630
631 631 if lines[-1].endswith('\\'):
632 632 # Explicit backslash continuation
633 633 return 'incomplete', find_last_indent(lines)
634 634
635 635 try:
636 636 for transform in self.cleanup_transforms:
637 637 lines = transform(lines)
638 638 except SyntaxError:
639 639 return 'invalid', None
640 640
641 641 if lines[0].startswith('%%'):
642 642 # Special case for cell magics - completion marked by blank line
643 643 if lines[-1].strip():
644 644 return 'incomplete', find_last_indent(lines)
645 645 else:
646 646 return 'complete', None
647 647
648 648 try:
649 649 for transform in self.line_transforms:
650 650 lines = transform(lines)
651 651 lines = self.do_token_transforms(lines)
652 652 except SyntaxError:
653 653 return 'invalid', None
654 654
655 655 tokens_by_line = make_tokens_by_line(lines)
656 656
657 657 if not tokens_by_line:
658 658 return 'incomplete', find_last_indent(lines)
659 659
660 660 if tokens_by_line[-1][-1].type != tokenize.ENDMARKER:
661 661 # We're in a multiline string or expression
662 662 return 'incomplete', find_last_indent(lines)
663 663
664 664 newline_types = {tokenize.NEWLINE, tokenize.COMMENT, tokenize.ENDMARKER}
665 665
666 666 # Pop the last line which only contains DEDENTs and ENDMARKER
667 667 last_token_line = None
668 668 if {t.type for t in tokens_by_line[-1]} in [
669 669 {tokenize.DEDENT, tokenize.ENDMARKER},
670 670 {tokenize.ENDMARKER}
671 671 ] and len(tokens_by_line) > 1:
672 672 last_token_line = tokens_by_line.pop()
673 673
674 674 while tokens_by_line[-1] and tokens_by_line[-1][-1].type in newline_types:
675 675 tokens_by_line[-1].pop()
676 676
677 677 if not tokens_by_line[-1]:
678 678 return 'incomplete', find_last_indent(lines)
679 679
680 680 if tokens_by_line[-1][-1].string == ':':
681 681 # The last line starts a block (e.g. 'if foo:')
682 682 ix = 0
683 683 while tokens_by_line[-1][ix].type in {tokenize.INDENT, tokenize.DEDENT}:
684 684 ix += 1
685 685
686 686 indent = tokens_by_line[-1][ix].start[1]
687 687 return 'incomplete', indent + 4
688 688
689 689 if tokens_by_line[-1][0].line.endswith('\\'):
690 690 return 'incomplete', None
691 691
692 692 # At this point, our checks think the code is complete (or invalid).
693 693 # We'll use codeop.compile_command to check this with the real parser
694 694 try:
695 695 with warnings.catch_warnings():
696 696 warnings.simplefilter('error', SyntaxWarning)
697 697 res = compile_command(''.join(lines), symbol='exec')
698 698 except (SyntaxError, OverflowError, ValueError, TypeError,
699 699 MemoryError, SyntaxWarning):
700 700 return 'invalid', None
701 701 else:
702 702 if res is None:
703 703 return 'incomplete', find_last_indent(lines)
704 704
705 705 if last_token_line and last_token_line[0].type == tokenize.DEDENT:
706 706 if ends_with_newline:
707 707 return 'complete', None
708 708 return 'incomplete', find_last_indent(lines)
709 709
710 710 # If there's a blank line at the end, assume we're ready to execute
711 711 if not lines[-1].strip():
712 712 return 'complete', None
713 713
714 714 return 'complete', None
715 715
716 716
717 717 def find_last_indent(lines):
718 718 m = _indent_re.match(lines[-1])
719 719 if not m:
720 720 return 0
721 721 return len(m.group(0).replace('\t', ' '*4))
@@ -1,714 +1,708 b''
1 1 """Implementation of namespace-related magic functions.
2 2 """
3 3 #-----------------------------------------------------------------------------
4 4 # Copyright (c) 2012 The IPython Development Team.
5 5 #
6 6 # Distributed under the terms of the Modified BSD License.
7 7 #
8 8 # The full license is in the file COPYING.txt, distributed with this software.
9 9 #-----------------------------------------------------------------------------
10 10
11 11 #-----------------------------------------------------------------------------
12 12 # Imports
13 13 #-----------------------------------------------------------------------------
14 14
15 15 # Stdlib
16 16 import gc
17 17 import re
18 18 import sys
19 19
20 20 # Our own packages
21 21 from IPython.core import page
22 22 from IPython.core.error import StdinNotImplementedError, UsageError
23 23 from IPython.core.magic import Magics, magics_class, line_magic
24 24 from IPython.testing.skipdoctest import skip_doctest
25 25 from IPython.utils.encoding import DEFAULT_ENCODING
26 26 from IPython.utils.openpy import read_py_file
27 27 from IPython.utils.path import get_py_filename
28 28
29 29 #-----------------------------------------------------------------------------
30 30 # Magic implementation classes
31 31 #-----------------------------------------------------------------------------
32 32
33 33 @magics_class
34 34 class NamespaceMagics(Magics):
35 35 """Magics to manage various aspects of the user's namespace.
36 36
37 37 These include listing variables, introspecting into them, etc.
38 38 """
39 39
40 40 @line_magic
41 41 def pinfo(self, parameter_s='', namespaces=None):
42 42 """Provide detailed information about an object.
43 43
44 44 '%pinfo object' is just a synonym for object? or ?object."""
45 45
46 46 #print 'pinfo par: <%s>' % parameter_s # dbg
47 47 # detail_level: 0 -> obj? , 1 -> obj??
48 48 detail_level = 0
49 49 # We need to detect if we got called as 'pinfo pinfo foo', which can
50 50 # happen if the user types 'pinfo foo?' at the cmd line.
51 51 pinfo,qmark1,oname,qmark2 = \
52 52 re.match(r'(pinfo )?(\?*)(.*?)(\??$)',parameter_s).groups()
53 53 if pinfo or qmark1 or qmark2:
54 54 detail_level = 1
55 55 if "*" in oname:
56 56 self.psearch(oname)
57 57 else:
58 58 self.shell._inspect('pinfo', oname, detail_level=detail_level,
59 59 namespaces=namespaces)
60 60
61 61 @line_magic
62 62 def pinfo2(self, parameter_s='', namespaces=None):
63 63 """Provide extra detailed information about an object.
64 64
65 65 '%pinfo2 object' is just a synonym for object?? or ??object."""
66 66 self.shell._inspect('pinfo', parameter_s, detail_level=1,
67 67 namespaces=namespaces)
68 68
69 69 @skip_doctest
70 70 @line_magic
71 71 def pdef(self, parameter_s='', namespaces=None):
72 72 """Print the call signature for any callable object.
73 73
74 74 If the object is a class, print the constructor information.
75 75
76 76 Examples
77 77 --------
78 78 ::
79 79
80 80 In [3]: %pdef urllib.urlopen
81 81 urllib.urlopen(url, data=None, proxies=None)
82 82 """
83 83 self.shell._inspect('pdef',parameter_s, namespaces)
84 84
85 85 @line_magic
86 86 def pdoc(self, parameter_s='', namespaces=None):
87 87 """Print the docstring for an object.
88 88
89 89 If the given object is a class, it will print both the class and the
90 90 constructor docstrings."""
91 91 self.shell._inspect('pdoc',parameter_s, namespaces)
92 92
93 93 @line_magic
94 94 def psource(self, parameter_s='', namespaces=None):
95 95 """Print (or run through pager) the source code for an object."""
96 96 if not parameter_s:
97 97 raise UsageError('Missing object name.')
98 98 self.shell._inspect('psource',parameter_s, namespaces)
99 99
100 100 @line_magic
101 101 def pfile(self, parameter_s='', namespaces=None):
102 102 """Print (or run through pager) the file where an object is defined.
103 103
104 104 The file opens at the line where the object definition begins. IPython
105 105 will honor the environment variable PAGER if set, and otherwise will
106 106 do its best to print the file in a convenient form.
107 107
108 108 If the given argument is not an object currently defined, IPython will
109 109 try to interpret it as a filename (automatically adding a .py extension
110 110 if needed). You can thus use %pfile as a syntax highlighting code
111 111 viewer."""
112 112
113 113 # first interpret argument as an object name
114 114 out = self.shell._inspect('pfile',parameter_s, namespaces)
115 115 # if not, try the input as a filename
116 116 if out == 'not found':
117 117 try:
118 118 filename = get_py_filename(parameter_s)
119 119 except IOError as msg:
120 120 print(msg)
121 121 return
122 122 page.page(self.shell.pycolorize(read_py_file(filename, skip_encoding_cookie=False)))
123 123
124 124 @line_magic
125 125 def psearch(self, parameter_s=''):
126 126 """Search for object in namespaces by wildcard.
127 127
128 128 %psearch [options] PATTERN [OBJECT TYPE]
129 129
130 130 Note: ? can be used as a synonym for %psearch, at the beginning or at
131 131 the end: both a*? and ?a* are equivalent to '%psearch a*'. Still, the
132 132 rest of the command line must be unchanged (options come first), so
133 133 for example the following forms are equivalent
134 134
135 135 %psearch -i a* function
136 136 -i a* function?
137 137 ?-i a* function
138 138
139 139 Arguments:
140 140
141 141 PATTERN
142 142
143 143 where PATTERN is a string containing * as a wildcard similar to its
144 144 use in a shell. The pattern is matched in all namespaces on the
145 145 search path. By default objects starting with a single _ are not
146 146 matched, many IPython generated objects have a single
147 147 underscore. The default is case insensitive matching. Matching is
148 148 also done on the attributes of objects and not only on the objects
149 149 in a module.
150 150
151 151 [OBJECT TYPE]
152 152
153 153 Is the name of a python type from the types module. The name is
154 154 given in lowercase without the ending type, ex. StringType is
155 155 written string. By adding a type here only objects matching the
156 156 given type are matched. Using all here makes the pattern match all
157 157 types (this is the default).
158 158
159 159 Options:
160 160
161 161 -a: makes the pattern match even objects whose names start with a
162 162 single underscore. These names are normally omitted from the
163 163 search.
164 164
165 165 -i/-c: make the pattern case insensitive/sensitive. If neither of
166 166 these options are given, the default is read from your configuration
167 167 file, with the option ``InteractiveShell.wildcards_case_sensitive``.
168 168 If this option is not specified in your configuration file, IPython's
169 169 internal default is to do a case sensitive search.
170 170
171 171 -e/-s NAMESPACE: exclude/search a given namespace. The pattern you
172 172 specify can be searched in any of the following namespaces:
173 173 'builtin', 'user', 'user_global','internal', 'alias', where
174 174 'builtin' and 'user' are the search defaults. Note that you should
175 175 not use quotes when specifying namespaces.
176 176
177 177 -l: List all available object types for object matching. This function
178 178 can be used without arguments.
179 179
180 180 'Builtin' contains the python module builtin, 'user' contains all
181 181 user data, 'alias' only contain the shell aliases and no python
182 182 objects, 'internal' contains objects used by IPython. The
183 183 'user_global' namespace is only used by embedded IPython instances,
184 184 and it contains module-level globals. You can add namespaces to the
185 185 search with -s or exclude them with -e (these options can be given
186 186 more than once).
187 187
188 188 Examples
189 189 --------
190 190 ::
191 191
192 192 %psearch a* -> objects beginning with an a
193 193 %psearch -e builtin a* -> objects NOT in the builtin space starting in a
194 194 %psearch a* function -> all functions beginning with an a
195 195 %psearch re.e* -> objects beginning with an e in module re
196 196 %psearch r*.e* -> objects that start with e in modules starting in r
197 197 %psearch r*.* string -> all strings in modules beginning with r
198 198
199 199 Case sensitive search::
200 200
201 201 %psearch -c a* list all object beginning with lower case a
202 202
203 203 Show objects beginning with a single _::
204 204
205 205 %psearch -a _* list objects beginning with a single underscore
206 206
207 207 List available objects::
208 208
209 209 %psearch -l list all available object types
210 210 """
211 try:
212 parameter_s.encode('ascii')
213 except UnicodeEncodeError:
214 print('Python identifiers can only contain ascii characters.')
215 return
216
217 211 # default namespaces to be searched
218 212 def_search = ['user_local', 'user_global', 'builtin']
219 213
220 214 # Process options/args
221 215 opts,args = self.parse_options(parameter_s,'cias:e:l',list_all=True)
222 216 opt = opts.get
223 217 shell = self.shell
224 218 psearch = shell.inspector.psearch
225 219
226 220 # select list object types
227 221 list_types = False
228 222 if 'l' in opts:
229 223 list_types = True
230 224
231 225 # select case options
232 226 if 'i' in opts:
233 227 ignore_case = True
234 228 elif 'c' in opts:
235 229 ignore_case = False
236 230 else:
237 231 ignore_case = not shell.wildcards_case_sensitive
238 232
239 233 # Build list of namespaces to search from user options
240 234 def_search.extend(opt('s',[]))
241 235 ns_exclude = ns_exclude=opt('e',[])
242 236 ns_search = [nm for nm in def_search if nm not in ns_exclude]
243 237
244 238 # Call the actual search
245 239 try:
246 240 psearch(args,shell.ns_table,ns_search,
247 241 show_all=opt('a'),ignore_case=ignore_case, list_types=list_types)
248 242 except:
249 243 shell.showtraceback()
250 244
251 245 @skip_doctest
252 246 @line_magic
253 247 def who_ls(self, parameter_s=''):
254 248 """Return a sorted list of all interactive variables.
255 249
256 250 If arguments are given, only variables of types matching these
257 251 arguments are returned.
258 252
259 253 Examples
260 254 --------
261 255
262 256 Define two variables and list them with who_ls::
263 257
264 258 In [1]: alpha = 123
265 259
266 260 In [2]: beta = 'test'
267 261
268 262 In [3]: %who_ls
269 263 Out[3]: ['alpha', 'beta']
270 264
271 265 In [4]: %who_ls int
272 266 Out[4]: ['alpha']
273 267
274 268 In [5]: %who_ls str
275 269 Out[5]: ['beta']
276 270 """
277 271
278 272 user_ns = self.shell.user_ns
279 273 user_ns_hidden = self.shell.user_ns_hidden
280 274 nonmatching = object() # This can never be in user_ns
281 275 out = [ i for i in user_ns
282 276 if not i.startswith('_') \
283 277 and (user_ns[i] is not user_ns_hidden.get(i, nonmatching)) ]
284 278
285 279 typelist = parameter_s.split()
286 280 if typelist:
287 281 typeset = set(typelist)
288 282 out = [i for i in out if type(user_ns[i]).__name__ in typeset]
289 283
290 284 out.sort()
291 285 return out
292 286
293 287 @skip_doctest
294 288 @line_magic
295 289 def who(self, parameter_s=''):
296 290 """Print all interactive variables, with some minimal formatting.
297 291
298 292 If any arguments are given, only variables whose type matches one of
299 293 these are printed. For example::
300 294
301 295 %who function str
302 296
303 297 will only list functions and strings, excluding all other types of
304 298 variables. To find the proper type names, simply use type(var) at a
305 299 command line to see how python prints type names. For example:
306 300
307 301 ::
308 302
309 303 In [1]: type('hello')\\
310 304 Out[1]: <type 'str'>
311 305
312 306 indicates that the type name for strings is 'str'.
313 307
314 308 ``%who`` always excludes executed names loaded through your configuration
315 309 file and things which are internal to IPython.
316 310
317 311 This is deliberate, as typically you may load many modules and the
318 312 purpose of %who is to show you only what you've manually defined.
319 313
320 314 Examples
321 315 --------
322 316
323 317 Define two variables and list them with who::
324 318
325 319 In [1]: alpha = 123
326 320
327 321 In [2]: beta = 'test'
328 322
329 323 In [3]: %who
330 324 alpha beta
331 325
332 326 In [4]: %who int
333 327 alpha
334 328
335 329 In [5]: %who str
336 330 beta
337 331 """
338 332
339 333 varlist = self.who_ls(parameter_s)
340 334 if not varlist:
341 335 if parameter_s:
342 336 print('No variables match your requested type.')
343 337 else:
344 338 print('Interactive namespace is empty.')
345 339 return
346 340
347 341 # if we have variables, move on...
348 342 count = 0
349 343 for i in varlist:
350 344 print(i+'\t', end=' ')
351 345 count += 1
352 346 if count > 8:
353 347 count = 0
354 348 print()
355 349 print()
356 350
357 351 @skip_doctest
358 352 @line_magic
359 353 def whos(self, parameter_s=''):
360 354 """Like %who, but gives some extra information about each variable.
361 355
362 356 The same type filtering of %who can be applied here.
363 357
364 358 For all variables, the type is printed. Additionally it prints:
365 359
366 360 - For {},[],(): their length.
367 361
368 362 - For numpy arrays, a summary with shape, number of
369 363 elements, typecode and size in memory.
370 364
371 365 - Everything else: a string representation, snipping their middle if
372 366 too long.
373 367
374 368 Examples
375 369 --------
376 370
377 371 Define two variables and list them with whos::
378 372
379 373 In [1]: alpha = 123
380 374
381 375 In [2]: beta = 'test'
382 376
383 377 In [3]: %whos
384 378 Variable Type Data/Info
385 379 --------------------------------
386 380 alpha int 123
387 381 beta str test
388 382 """
389 383
390 384 varnames = self.who_ls(parameter_s)
391 385 if not varnames:
392 386 if parameter_s:
393 387 print('No variables match your requested type.')
394 388 else:
395 389 print('Interactive namespace is empty.')
396 390 return
397 391
398 392 # if we have variables, move on...
399 393
400 394 # for these types, show len() instead of data:
401 395 seq_types = ['dict', 'list', 'tuple']
402 396
403 397 # for numpy arrays, display summary info
404 398 ndarray_type = None
405 399 if 'numpy' in sys.modules:
406 400 try:
407 401 from numpy import ndarray
408 402 except ImportError:
409 403 pass
410 404 else:
411 405 ndarray_type = ndarray.__name__
412 406
413 407 # Find all variable names and types so we can figure out column sizes
414 408
415 409 # some types are well known and can be shorter
416 410 abbrevs = {'IPython.core.macro.Macro' : 'Macro'}
417 411 def type_name(v):
418 412 tn = type(v).__name__
419 413 return abbrevs.get(tn,tn)
420 414
421 415 varlist = [self.shell.user_ns[n] for n in varnames]
422 416
423 417 typelist = []
424 418 for vv in varlist:
425 419 tt = type_name(vv)
426 420
427 421 if tt=='instance':
428 422 typelist.append( abbrevs.get(str(vv.__class__),
429 423 str(vv.__class__)))
430 424 else:
431 425 typelist.append(tt)
432 426
433 427 # column labels and # of spaces as separator
434 428 varlabel = 'Variable'
435 429 typelabel = 'Type'
436 430 datalabel = 'Data/Info'
437 431 colsep = 3
438 432 # variable format strings
439 433 vformat = "{0:<{varwidth}}{1:<{typewidth}}"
440 434 aformat = "%s: %s elems, type `%s`, %s bytes"
441 435 # find the size of the columns to format the output nicely
442 436 varwidth = max(max(map(len,varnames)), len(varlabel)) + colsep
443 437 typewidth = max(max(map(len,typelist)), len(typelabel)) + colsep
444 438 # table header
445 439 print(varlabel.ljust(varwidth) + typelabel.ljust(typewidth) + \
446 440 ' '+datalabel+'\n' + '-'*(varwidth+typewidth+len(datalabel)+1))
447 441 # and the table itself
448 442 kb = 1024
449 443 Mb = 1048576 # kb**2
450 444 for vname,var,vtype in zip(varnames,varlist,typelist):
451 445 print(vformat.format(vname, vtype, varwidth=varwidth, typewidth=typewidth), end=' ')
452 446 if vtype in seq_types:
453 447 print("n="+str(len(var)))
454 448 elif vtype == ndarray_type:
455 449 vshape = str(var.shape).replace(',','').replace(' ','x')[1:-1]
456 450 if vtype==ndarray_type:
457 451 # numpy
458 452 vsize = var.size
459 453 vbytes = vsize*var.itemsize
460 454 vdtype = var.dtype
461 455
462 456 if vbytes < 100000:
463 457 print(aformat % (vshape, vsize, vdtype, vbytes))
464 458 else:
465 459 print(aformat % (vshape, vsize, vdtype, vbytes), end=' ')
466 460 if vbytes < Mb:
467 461 print('(%s kb)' % (vbytes/kb,))
468 462 else:
469 463 print('(%s Mb)' % (vbytes/Mb,))
470 464 else:
471 465 try:
472 466 vstr = str(var)
473 467 except UnicodeEncodeError:
474 468 vstr = var.encode(DEFAULT_ENCODING,
475 469 'backslashreplace')
476 470 except:
477 471 vstr = "<object with id %d (str() failed)>" % id(var)
478 472 vstr = vstr.replace('\n', '\\n')
479 473 if len(vstr) < 50:
480 474 print(vstr)
481 475 else:
482 476 print(vstr[:25] + "<...>" + vstr[-25:])
483 477
484 478 @line_magic
485 479 def reset(self, parameter_s=''):
486 480 """Resets the namespace by removing all names defined by the user, if
487 481 called without arguments, or by removing some types of objects, such
488 482 as everything currently in IPython's In[] and Out[] containers (see
489 483 the parameters for details).
490 484
491 485 Parameters
492 486 ----------
493 487 -f : force reset without asking for confirmation.
494 488
495 489 -s : 'Soft' reset: Only clears your namespace, leaving history intact.
496 490 References to objects may be kept. By default (without this option),
497 491 we do a 'hard' reset, giving you a new session and removing all
498 492 references to objects from the current session.
499 493
500 494 in : reset input history
501 495
502 496 out : reset output history
503 497
504 498 dhist : reset directory history
505 499
506 500 array : reset only variables that are NumPy arrays
507 501
508 502 See Also
509 503 --------
510 504 reset_selective : invoked as ``%reset_selective``
511 505
512 506 Examples
513 507 --------
514 508 ::
515 509
516 510 In [6]: a = 1
517 511
518 512 In [7]: a
519 513 Out[7]: 1
520 514
521 515 In [8]: 'a' in get_ipython().user_ns
522 516 Out[8]: True
523 517
524 518 In [9]: %reset -f
525 519
526 520 In [1]: 'a' in get_ipython().user_ns
527 521 Out[1]: False
528 522
529 523 In [2]: %reset -f in
530 524 Flushing input history
531 525
532 526 In [3]: %reset -f dhist in
533 527 Flushing directory history
534 528 Flushing input history
535 529
536 530 Notes
537 531 -----
538 532 Calling this magic from clients that do not implement standard input,
539 533 such as the ipython notebook interface, will reset the namespace
540 534 without confirmation.
541 535 """
542 536 opts, args = self.parse_options(parameter_s,'sf', mode='list')
543 537 if 'f' in opts:
544 538 ans = True
545 539 else:
546 540 try:
547 541 ans = self.shell.ask_yes_no(
548 542 "Once deleted, variables cannot be recovered. Proceed (y/[n])?",
549 543 default='n')
550 544 except StdinNotImplementedError:
551 545 ans = True
552 546 if not ans:
553 547 print('Nothing done.')
554 548 return
555 549
556 550 if 's' in opts: # Soft reset
557 551 user_ns = self.shell.user_ns
558 552 for i in self.who_ls():
559 553 del(user_ns[i])
560 554 elif len(args) == 0: # Hard reset
561 555 self.shell.reset(new_session = False)
562 556
563 557 # reset in/out/dhist/array: previously extensinions/clearcmd.py
564 558 ip = self.shell
565 559 user_ns = self.shell.user_ns # local lookup, heavily used
566 560
567 561 for target in args:
568 562 target = target.lower() # make matches case insensitive
569 563 if target == 'out':
570 564 print("Flushing output cache (%d entries)" % len(user_ns['_oh']))
571 565 self.shell.displayhook.flush()
572 566
573 567 elif target == 'in':
574 568 print("Flushing input history")
575 569 pc = self.shell.displayhook.prompt_count + 1
576 570 for n in range(1, pc):
577 571 key = '_i'+repr(n)
578 572 user_ns.pop(key,None)
579 573 user_ns.update(dict(_i=u'',_ii=u'',_iii=u''))
580 574 hm = ip.history_manager
581 575 # don't delete these, as %save and %macro depending on the
582 576 # length of these lists to be preserved
583 577 hm.input_hist_parsed[:] = [''] * pc
584 578 hm.input_hist_raw[:] = [''] * pc
585 579 # hm has internal machinery for _i,_ii,_iii, clear it out
586 580 hm._i = hm._ii = hm._iii = hm._i00 = u''
587 581
588 582 elif target == 'array':
589 583 # Support cleaning up numpy arrays
590 584 try:
591 585 from numpy import ndarray
592 586 # This must be done with items and not iteritems because
593 587 # we're going to modify the dict in-place.
594 588 for x,val in list(user_ns.items()):
595 589 if isinstance(val,ndarray):
596 590 del user_ns[x]
597 591 except ImportError:
598 592 print("reset array only works if Numpy is available.")
599 593
600 594 elif target == 'dhist':
601 595 print("Flushing directory history")
602 596 del user_ns['_dh'][:]
603 597
604 598 else:
605 599 print("Don't know how to reset ", end=' ')
606 600 print(target + ", please run `%reset?` for details")
607 601
608 602 gc.collect()
609 603
610 604 @line_magic
611 605 def reset_selective(self, parameter_s=''):
612 606 """Resets the namespace by removing names defined by the user.
613 607
614 608 Input/Output history are left around in case you need them.
615 609
616 610 %reset_selective [-f] regex
617 611
618 612 No action is taken if regex is not included
619 613
620 614 Options
621 615 -f : force reset without asking for confirmation.
622 616
623 617 See Also
624 618 --------
625 619 reset : invoked as ``%reset``
626 620
627 621 Examples
628 622 --------
629 623
630 624 We first fully reset the namespace so your output looks identical to
631 625 this example for pedagogical reasons; in practice you do not need a
632 626 full reset::
633 627
634 628 In [1]: %reset -f
635 629
636 630 Now, with a clean namespace we can make a few variables and use
637 631 ``%reset_selective`` to only delete names that match our regexp::
638 632
639 633 In [2]: a=1; b=2; c=3; b1m=4; b2m=5; b3m=6; b4m=7; b2s=8
640 634
641 635 In [3]: who_ls
642 636 Out[3]: ['a', 'b', 'b1m', 'b2m', 'b2s', 'b3m', 'b4m', 'c']
643 637
644 638 In [4]: %reset_selective -f b[2-3]m
645 639
646 640 In [5]: who_ls
647 641 Out[5]: ['a', 'b', 'b1m', 'b2s', 'b4m', 'c']
648 642
649 643 In [6]: %reset_selective -f d
650 644
651 645 In [7]: who_ls
652 646 Out[7]: ['a', 'b', 'b1m', 'b2s', 'b4m', 'c']
653 647
654 648 In [8]: %reset_selective -f c
655 649
656 650 In [9]: who_ls
657 651 Out[9]: ['a', 'b', 'b1m', 'b2s', 'b4m']
658 652
659 653 In [10]: %reset_selective -f b
660 654
661 655 In [11]: who_ls
662 656 Out[11]: ['a']
663 657
664 658 Notes
665 659 -----
666 660 Calling this magic from clients that do not implement standard input,
667 661 such as the ipython notebook interface, will reset the namespace
668 662 without confirmation.
669 663 """
670 664
671 665 opts, regex = self.parse_options(parameter_s,'f')
672 666
673 667 if 'f' in opts:
674 668 ans = True
675 669 else:
676 670 try:
677 671 ans = self.shell.ask_yes_no(
678 672 "Once deleted, variables cannot be recovered. Proceed (y/[n])? ",
679 673 default='n')
680 674 except StdinNotImplementedError:
681 675 ans = True
682 676 if not ans:
683 677 print('Nothing done.')
684 678 return
685 679 user_ns = self.shell.user_ns
686 680 if not regex:
687 681 print('No regex pattern specified. Nothing done.')
688 682 return
689 683 else:
690 684 try:
691 685 m = re.compile(regex)
692 686 except TypeError:
693 687 raise TypeError('regex must be a string or compiled pattern')
694 688 for i in self.who_ls():
695 689 if m.search(i):
696 690 del(user_ns[i])
697 691
698 692 @line_magic
699 693 def xdel(self, parameter_s=''):
700 694 """Delete a variable, trying to clear it from anywhere that
701 695 IPython's machinery has references to it. By default, this uses
702 696 the identity of the named object in the user namespace to remove
703 697 references held under other names. The object is also removed
704 698 from the output history.
705 699
706 700 Options
707 701 -n : Delete the specified name from all namespaces, without
708 702 checking their identity.
709 703 """
710 704 opts, varname = self.parse_options(parameter_s,'n')
711 705 try:
712 706 self.shell.del_var(varname, ('n' in opts))
713 707 except (NameError, ValueError) as e:
714 708 print(type(e).__name__ +": "+ str(e))
@@ -1,709 +1,709 b''
1 1 # encoding: utf-8
2 2 """
3 3 Prefiltering components.
4 4
5 5 Prefilters transform user input before it is exec'd by Python. These
6 6 transforms are used to implement additional syntax such as !ls and %magic.
7 7 """
8 8
9 9 # Copyright (c) IPython Development Team.
10 10 # Distributed under the terms of the Modified BSD License.
11 11
12 12 from keyword import iskeyword
13 13 import re
14 14
15 15 from .autocall import IPyAutocall
16 16 from traitlets.config.configurable import Configurable
17 17 from .inputtransformer2 import (
18 18 ESC_MAGIC,
19 19 ESC_QUOTE,
20 20 ESC_QUOTE2,
21 21 ESC_PAREN,
22 22 )
23 23 from .macro import Macro
24 24 from .splitinput import LineInfo
25 25
26 26 from traitlets import (
27 27 List, Integer, Unicode, Bool, Instance, CRegExp
28 28 )
29 29
30 30 #-----------------------------------------------------------------------------
31 31 # Global utilities, errors and constants
32 32 #-----------------------------------------------------------------------------
33 33
34 34
35 35 class PrefilterError(Exception):
36 36 pass
37 37
38 38
39 39 # RegExp to identify potential function names
40 re_fun_name = re.compile(r'[a-zA-Z_]([a-zA-Z0-9_.]*) *$')
40 re_fun_name = re.compile(r'[^\W\d]([\w.]*) *$')
41 41
42 42 # RegExp to exclude strings with this start from autocalling. In
43 43 # particular, all binary operators should be excluded, so that if foo is
44 44 # callable, foo OP bar doesn't become foo(OP bar), which is invalid. The
45 45 # characters '!=()' don't need to be checked for, as the checkPythonChars
46 46 # routine explicitly does so, to catch direct calls and rebindings of
47 47 # existing names.
48 48
49 49 # Warning: the '-' HAS TO BE AT THE END of the first group, otherwise
50 50 # it affects the rest of the group in square brackets.
51 51 re_exclude_auto = re.compile(r'^[,&^\|\*/\+-]'
52 52 r'|^is |^not |^in |^and |^or ')
53 53
54 54 # try to catch also methods for stuff in lists/tuples/dicts: off
55 55 # (experimental). For this to work, the line_split regexp would need
56 56 # to be modified so it wouldn't break things at '['. That line is
57 57 # nasty enough that I shouldn't change it until I can test it _well_.
58 58 #self.re_fun_name = re.compile (r'[a-zA-Z_]([a-zA-Z0-9_.\[\]]*) ?$')
59 59
60 60
61 61 # Handler Check Utilities
62 62 def is_shadowed(identifier, ip):
63 63 """Is the given identifier defined in one of the namespaces which shadow
64 64 the alias and magic namespaces? Note that an identifier is different
65 65 than ifun, because it can not contain a '.' character."""
66 66 # This is much safer than calling ofind, which can change state
67 67 return (identifier in ip.user_ns \
68 68 or identifier in ip.user_global_ns \
69 69 or identifier in ip.ns_table['builtin']\
70 70 or iskeyword(identifier))
71 71
72 72
73 73 #-----------------------------------------------------------------------------
74 74 # Main Prefilter manager
75 75 #-----------------------------------------------------------------------------
76 76
77 77
78 78 class PrefilterManager(Configurable):
79 79 """Main prefilter component.
80 80
81 81 The IPython prefilter is run on all user input before it is run. The
82 82 prefilter consumes lines of input and produces transformed lines of
83 83 input.
84 84
85 85 The implementation consists of two phases:
86 86
87 87 1. Transformers
88 88 2. Checkers and handlers
89 89
90 90 Over time, we plan on deprecating the checkers and handlers and doing
91 91 everything in the transformers.
92 92
93 93 The transformers are instances of :class:`PrefilterTransformer` and have
94 94 a single method :meth:`transform` that takes a line and returns a
95 95 transformed line. The transformation can be accomplished using any
96 96 tool, but our current ones use regular expressions for speed.
97 97
98 98 After all the transformers have been run, the line is fed to the checkers,
99 99 which are instances of :class:`PrefilterChecker`. The line is passed to
100 100 the :meth:`check` method, which either returns `None` or a
101 101 :class:`PrefilterHandler` instance. If `None` is returned, the other
102 102 checkers are tried. If an :class:`PrefilterHandler` instance is returned,
103 103 the line is passed to the :meth:`handle` method of the returned
104 104 handler and no further checkers are tried.
105 105
106 106 Both transformers and checkers have a `priority` attribute, that determines
107 107 the order in which they are called. Smaller priorities are tried first.
108 108
109 109 Both transformers and checkers also have `enabled` attribute, which is
110 110 a boolean that determines if the instance is used.
111 111
112 112 Users or developers can change the priority or enabled attribute of
113 113 transformers or checkers, but they must call the :meth:`sort_checkers`
114 114 or :meth:`sort_transformers` method after changing the priority.
115 115 """
116 116
117 117 multi_line_specials = Bool(True).tag(config=True)
118 118 shell = Instance('IPython.core.interactiveshell.InteractiveShellABC', allow_none=True)
119 119
120 120 def __init__(self, shell=None, **kwargs):
121 121 super(PrefilterManager, self).__init__(shell=shell, **kwargs)
122 122 self.shell = shell
123 123 self.init_transformers()
124 124 self.init_handlers()
125 125 self.init_checkers()
126 126
127 127 #-------------------------------------------------------------------------
128 128 # API for managing transformers
129 129 #-------------------------------------------------------------------------
130 130
131 131 def init_transformers(self):
132 132 """Create the default transformers."""
133 133 self._transformers = []
134 134 for transformer_cls in _default_transformers:
135 135 transformer_cls(
136 136 shell=self.shell, prefilter_manager=self, parent=self
137 137 )
138 138
139 139 def sort_transformers(self):
140 140 """Sort the transformers by priority.
141 141
142 142 This must be called after the priority of a transformer is changed.
143 143 The :meth:`register_transformer` method calls this automatically.
144 144 """
145 145 self._transformers.sort(key=lambda x: x.priority)
146 146
147 147 @property
148 148 def transformers(self):
149 149 """Return a list of checkers, sorted by priority."""
150 150 return self._transformers
151 151
152 152 def register_transformer(self, transformer):
153 153 """Register a transformer instance."""
154 154 if transformer not in self._transformers:
155 155 self._transformers.append(transformer)
156 156 self.sort_transformers()
157 157
158 158 def unregister_transformer(self, transformer):
159 159 """Unregister a transformer instance."""
160 160 if transformer in self._transformers:
161 161 self._transformers.remove(transformer)
162 162
163 163 #-------------------------------------------------------------------------
164 164 # API for managing checkers
165 165 #-------------------------------------------------------------------------
166 166
167 167 def init_checkers(self):
168 168 """Create the default checkers."""
169 169 self._checkers = []
170 170 for checker in _default_checkers:
171 171 checker(
172 172 shell=self.shell, prefilter_manager=self, parent=self
173 173 )
174 174
175 175 def sort_checkers(self):
176 176 """Sort the checkers by priority.
177 177
178 178 This must be called after the priority of a checker is changed.
179 179 The :meth:`register_checker` method calls this automatically.
180 180 """
181 181 self._checkers.sort(key=lambda x: x.priority)
182 182
183 183 @property
184 184 def checkers(self):
185 185 """Return a list of checkers, sorted by priority."""
186 186 return self._checkers
187 187
188 188 def register_checker(self, checker):
189 189 """Register a checker instance."""
190 190 if checker not in self._checkers:
191 191 self._checkers.append(checker)
192 192 self.sort_checkers()
193 193
194 194 def unregister_checker(self, checker):
195 195 """Unregister a checker instance."""
196 196 if checker in self._checkers:
197 197 self._checkers.remove(checker)
198 198
199 199 #-------------------------------------------------------------------------
200 200 # API for managing handlers
201 201 #-------------------------------------------------------------------------
202 202
203 203 def init_handlers(self):
204 204 """Create the default handlers."""
205 205 self._handlers = {}
206 206 self._esc_handlers = {}
207 207 for handler in _default_handlers:
208 208 handler(
209 209 shell=self.shell, prefilter_manager=self, parent=self
210 210 )
211 211
212 212 @property
213 213 def handlers(self):
214 214 """Return a dict of all the handlers."""
215 215 return self._handlers
216 216
217 217 def register_handler(self, name, handler, esc_strings):
218 218 """Register a handler instance by name with esc_strings."""
219 219 self._handlers[name] = handler
220 220 for esc_str in esc_strings:
221 221 self._esc_handlers[esc_str] = handler
222 222
223 223 def unregister_handler(self, name, handler, esc_strings):
224 224 """Unregister a handler instance by name with esc_strings."""
225 225 try:
226 226 del self._handlers[name]
227 227 except KeyError:
228 228 pass
229 229 for esc_str in esc_strings:
230 230 h = self._esc_handlers.get(esc_str)
231 231 if h is handler:
232 232 del self._esc_handlers[esc_str]
233 233
234 234 def get_handler_by_name(self, name):
235 235 """Get a handler by its name."""
236 236 return self._handlers.get(name)
237 237
238 238 def get_handler_by_esc(self, esc_str):
239 239 """Get a handler by its escape string."""
240 240 return self._esc_handlers.get(esc_str)
241 241
242 242 #-------------------------------------------------------------------------
243 243 # Main prefiltering API
244 244 #-------------------------------------------------------------------------
245 245
246 246 def prefilter_line_info(self, line_info):
247 247 """Prefilter a line that has been converted to a LineInfo object.
248 248
249 249 This implements the checker/handler part of the prefilter pipe.
250 250 """
251 251 # print "prefilter_line_info: ", line_info
252 252 handler = self.find_handler(line_info)
253 253 return handler.handle(line_info)
254 254
255 255 def find_handler(self, line_info):
256 256 """Find a handler for the line_info by trying checkers."""
257 257 for checker in self.checkers:
258 258 if checker.enabled:
259 259 handler = checker.check(line_info)
260 260 if handler:
261 261 return handler
262 262 return self.get_handler_by_name('normal')
263 263
264 264 def transform_line(self, line, continue_prompt):
265 265 """Calls the enabled transformers in order of increasing priority."""
266 266 for transformer in self.transformers:
267 267 if transformer.enabled:
268 268 line = transformer.transform(line, continue_prompt)
269 269 return line
270 270
271 271 def prefilter_line(self, line, continue_prompt=False):
272 272 """Prefilter a single input line as text.
273 273
274 274 This method prefilters a single line of text by calling the
275 275 transformers and then the checkers/handlers.
276 276 """
277 277
278 278 # print "prefilter_line: ", line, continue_prompt
279 279 # All handlers *must* return a value, even if it's blank ('').
280 280
281 281 # save the line away in case we crash, so the post-mortem handler can
282 282 # record it
283 283 self.shell._last_input_line = line
284 284
285 285 if not line:
286 286 # Return immediately on purely empty lines, so that if the user
287 287 # previously typed some whitespace that started a continuation
288 288 # prompt, he can break out of that loop with just an empty line.
289 289 # This is how the default python prompt works.
290 290 return ''
291 291
292 292 # At this point, we invoke our transformers.
293 293 if not continue_prompt or (continue_prompt and self.multi_line_specials):
294 294 line = self.transform_line(line, continue_prompt)
295 295
296 296 # Now we compute line_info for the checkers and handlers
297 297 line_info = LineInfo(line, continue_prompt)
298 298
299 299 # the input history needs to track even empty lines
300 300 stripped = line.strip()
301 301
302 302 normal_handler = self.get_handler_by_name('normal')
303 303 if not stripped:
304 304 return normal_handler.handle(line_info)
305 305
306 306 # special handlers are only allowed for single line statements
307 307 if continue_prompt and not self.multi_line_specials:
308 308 return normal_handler.handle(line_info)
309 309
310 310 prefiltered = self.prefilter_line_info(line_info)
311 311 # print "prefiltered line: %r" % prefiltered
312 312 return prefiltered
313 313
314 314 def prefilter_lines(self, lines, continue_prompt=False):
315 315 """Prefilter multiple input lines of text.
316 316
317 317 This is the main entry point for prefiltering multiple lines of
318 318 input. This simply calls :meth:`prefilter_line` for each line of
319 319 input.
320 320
321 321 This covers cases where there are multiple lines in the user entry,
322 322 which is the case when the user goes back to a multiline history
323 323 entry and presses enter.
324 324 """
325 325 llines = lines.rstrip('\n').split('\n')
326 326 # We can get multiple lines in one shot, where multiline input 'blends'
327 327 # into one line, in cases like recalling from the readline history
328 328 # buffer. We need to make sure that in such cases, we correctly
329 329 # communicate downstream which line is first and which are continuation
330 330 # ones.
331 331 if len(llines) > 1:
332 332 out = '\n'.join([self.prefilter_line(line, lnum>0)
333 333 for lnum, line in enumerate(llines) ])
334 334 else:
335 335 out = self.prefilter_line(llines[0], continue_prompt)
336 336
337 337 return out
338 338
339 339 #-----------------------------------------------------------------------------
340 340 # Prefilter transformers
341 341 #-----------------------------------------------------------------------------
342 342
343 343
344 344 class PrefilterTransformer(Configurable):
345 345 """Transform a line of user input."""
346 346
347 347 priority = Integer(100).tag(config=True)
348 348 # Transformers don't currently use shell or prefilter_manager, but as we
349 349 # move away from checkers and handlers, they will need them.
350 350 shell = Instance('IPython.core.interactiveshell.InteractiveShellABC', allow_none=True)
351 351 prefilter_manager = Instance('IPython.core.prefilter.PrefilterManager', allow_none=True)
352 352 enabled = Bool(True).tag(config=True)
353 353
354 354 def __init__(self, shell=None, prefilter_manager=None, **kwargs):
355 355 super(PrefilterTransformer, self).__init__(
356 356 shell=shell, prefilter_manager=prefilter_manager, **kwargs
357 357 )
358 358 self.prefilter_manager.register_transformer(self)
359 359
360 360 def transform(self, line, continue_prompt):
361 361 """Transform a line, returning the new one."""
362 362 return None
363 363
364 364 def __repr__(self):
365 365 return "<%s(priority=%r, enabled=%r)>" % (
366 366 self.__class__.__name__, self.priority, self.enabled)
367 367
368 368
369 369 #-----------------------------------------------------------------------------
370 370 # Prefilter checkers
371 371 #-----------------------------------------------------------------------------
372 372
373 373
374 374 class PrefilterChecker(Configurable):
375 375 """Inspect an input line and return a handler for that line."""
376 376
377 377 priority = Integer(100).tag(config=True)
378 378 shell = Instance('IPython.core.interactiveshell.InteractiveShellABC', allow_none=True)
379 379 prefilter_manager = Instance('IPython.core.prefilter.PrefilterManager', allow_none=True)
380 380 enabled = Bool(True).tag(config=True)
381 381
382 382 def __init__(self, shell=None, prefilter_manager=None, **kwargs):
383 383 super(PrefilterChecker, self).__init__(
384 384 shell=shell, prefilter_manager=prefilter_manager, **kwargs
385 385 )
386 386 self.prefilter_manager.register_checker(self)
387 387
388 388 def check(self, line_info):
389 389 """Inspect line_info and return a handler instance or None."""
390 390 return None
391 391
392 392 def __repr__(self):
393 393 return "<%s(priority=%r, enabled=%r)>" % (
394 394 self.__class__.__name__, self.priority, self.enabled)
395 395
396 396
397 397 class EmacsChecker(PrefilterChecker):
398 398
399 399 priority = Integer(100).tag(config=True)
400 400 enabled = Bool(False).tag(config=True)
401 401
402 402 def check(self, line_info):
403 403 "Emacs ipython-mode tags certain input lines."
404 404 if line_info.line.endswith('# PYTHON-MODE'):
405 405 return self.prefilter_manager.get_handler_by_name('emacs')
406 406 else:
407 407 return None
408 408
409 409
410 410 class MacroChecker(PrefilterChecker):
411 411
412 412 priority = Integer(250).tag(config=True)
413 413
414 414 def check(self, line_info):
415 415 obj = self.shell.user_ns.get(line_info.ifun)
416 416 if isinstance(obj, Macro):
417 417 return self.prefilter_manager.get_handler_by_name('macro')
418 418 else:
419 419 return None
420 420
421 421
422 422 class IPyAutocallChecker(PrefilterChecker):
423 423
424 424 priority = Integer(300).tag(config=True)
425 425
426 426 def check(self, line_info):
427 427 "Instances of IPyAutocall in user_ns get autocalled immediately"
428 428 obj = self.shell.user_ns.get(line_info.ifun, None)
429 429 if isinstance(obj, IPyAutocall):
430 430 obj.set_ip(self.shell)
431 431 return self.prefilter_manager.get_handler_by_name('auto')
432 432 else:
433 433 return None
434 434
435 435
436 436 class AssignmentChecker(PrefilterChecker):
437 437
438 438 priority = Integer(600).tag(config=True)
439 439
440 440 def check(self, line_info):
441 441 """Check to see if user is assigning to a var for the first time, in
442 442 which case we want to avoid any sort of automagic / autocall games.
443 443
444 444 This allows users to assign to either alias or magic names true python
445 445 variables (the magic/alias systems always take second seat to true
446 446 python code). E.g. ls='hi', or ls,that=1,2"""
447 447 if line_info.the_rest:
448 448 if line_info.the_rest[0] in '=,':
449 449 return self.prefilter_manager.get_handler_by_name('normal')
450 450 else:
451 451 return None
452 452
453 453
454 454 class AutoMagicChecker(PrefilterChecker):
455 455
456 456 priority = Integer(700).tag(config=True)
457 457
458 458 def check(self, line_info):
459 459 """If the ifun is magic, and automagic is on, run it. Note: normal,
460 460 non-auto magic would already have been triggered via '%' in
461 461 check_esc_chars. This just checks for automagic. Also, before
462 462 triggering the magic handler, make sure that there is nothing in the
463 463 user namespace which could shadow it."""
464 464 if not self.shell.automagic or not self.shell.find_magic(line_info.ifun):
465 465 return None
466 466
467 467 # We have a likely magic method. Make sure we should actually call it.
468 468 if line_info.continue_prompt and not self.prefilter_manager.multi_line_specials:
469 469 return None
470 470
471 471 head = line_info.ifun.split('.',1)[0]
472 472 if is_shadowed(head, self.shell):
473 473 return None
474 474
475 475 return self.prefilter_manager.get_handler_by_name('magic')
476 476
477 477
478 478 class PythonOpsChecker(PrefilterChecker):
479 479
480 480 priority = Integer(900).tag(config=True)
481 481
482 482 def check(self, line_info):
483 483 """If the 'rest' of the line begins with a function call or pretty much
484 484 any python operator, we should simply execute the line (regardless of
485 485 whether or not there's a possible autocall expansion). This avoids
486 486 spurious (and very confusing) geattr() accesses."""
487 487 if line_info.the_rest and line_info.the_rest[0] in '!=()<>,+*/%^&|':
488 488 return self.prefilter_manager.get_handler_by_name('normal')
489 489 else:
490 490 return None
491 491
492 492
493 493 class AutocallChecker(PrefilterChecker):
494 494
495 495 priority = Integer(1000).tag(config=True)
496 496
497 497 function_name_regexp = CRegExp(re_fun_name,
498 498 help="RegExp to identify potential function names."
499 499 ).tag(config=True)
500 500 exclude_regexp = CRegExp(re_exclude_auto,
501 501 help="RegExp to exclude strings with this start from autocalling."
502 502 ).tag(config=True)
503 503
504 504 def check(self, line_info):
505 505 "Check if the initial word/function is callable and autocall is on."
506 506 if not self.shell.autocall:
507 507 return None
508 508
509 509 oinfo = line_info.ofind(self.shell) # This can mutate state via getattr
510 510 if not oinfo['found']:
511 511 return None
512 512
513 513 ignored_funs = ['b', 'f', 'r', 'u', 'br', 'rb', 'fr', 'rf']
514 514 ifun = line_info.ifun
515 515 line = line_info.line
516 516 if ifun.lower() in ignored_funs and (line.startswith(ifun + "'") or line.startswith(ifun + '"')):
517 517 return None
518 518
519 519 if callable(oinfo['obj']) \
520 520 and (not self.exclude_regexp.match(line_info.the_rest)) \
521 521 and self.function_name_regexp.match(line_info.ifun):
522 522 return self.prefilter_manager.get_handler_by_name('auto')
523 523 else:
524 524 return None
525 525
526 526
527 527 #-----------------------------------------------------------------------------
528 528 # Prefilter handlers
529 529 #-----------------------------------------------------------------------------
530 530
531 531
532 532 class PrefilterHandler(Configurable):
533 533
534 534 handler_name = Unicode('normal')
535 535 esc_strings = List([])
536 536 shell = Instance('IPython.core.interactiveshell.InteractiveShellABC', allow_none=True)
537 537 prefilter_manager = Instance('IPython.core.prefilter.PrefilterManager', allow_none=True)
538 538
539 539 def __init__(self, shell=None, prefilter_manager=None, **kwargs):
540 540 super(PrefilterHandler, self).__init__(
541 541 shell=shell, prefilter_manager=prefilter_manager, **kwargs
542 542 )
543 543 self.prefilter_manager.register_handler(
544 544 self.handler_name,
545 545 self,
546 546 self.esc_strings
547 547 )
548 548
549 549 def handle(self, line_info):
550 550 # print "normal: ", line_info
551 551 """Handle normal input lines. Use as a template for handlers."""
552 552
553 553 # With autoindent on, we need some way to exit the input loop, and I
554 554 # don't want to force the user to have to backspace all the way to
555 555 # clear the line. The rule will be in this case, that either two
556 556 # lines of pure whitespace in a row, or a line of pure whitespace but
557 557 # of a size different to the indent level, will exit the input loop.
558 558 line = line_info.line
559 559 continue_prompt = line_info.continue_prompt
560 560
561 561 if (continue_prompt and
562 562 self.shell.autoindent and
563 563 line.isspace() and
564 564 0 < abs(len(line) - self.shell.indent_current_nsp) <= 2):
565 565 line = ''
566 566
567 567 return line
568 568
569 569 def __str__(self):
570 570 return "<%s(name=%s)>" % (self.__class__.__name__, self.handler_name)
571 571
572 572
573 573 class MacroHandler(PrefilterHandler):
574 574 handler_name = Unicode("macro")
575 575
576 576 def handle(self, line_info):
577 577 obj = self.shell.user_ns.get(line_info.ifun)
578 578 pre_space = line_info.pre_whitespace
579 579 line_sep = "\n" + pre_space
580 580 return pre_space + line_sep.join(obj.value.splitlines())
581 581
582 582
583 583 class MagicHandler(PrefilterHandler):
584 584
585 585 handler_name = Unicode('magic')
586 586 esc_strings = List([ESC_MAGIC])
587 587
588 588 def handle(self, line_info):
589 589 """Execute magic functions."""
590 590 ifun = line_info.ifun
591 591 the_rest = line_info.the_rest
592 592 #Prepare arguments for get_ipython().run_line_magic(magic_name, magic_args)
593 593 t_arg_s = ifun + " " + the_rest
594 594 t_magic_name, _, t_magic_arg_s = t_arg_s.partition(' ')
595 595 t_magic_name = t_magic_name.lstrip(ESC_MAGIC)
596 596 cmd = '%sget_ipython().run_line_magic(%r, %r)' % (line_info.pre_whitespace, t_magic_name, t_magic_arg_s)
597 597 return cmd
598 598
599 599
600 600 class AutoHandler(PrefilterHandler):
601 601
602 602 handler_name = Unicode('auto')
603 603 esc_strings = List([ESC_PAREN, ESC_QUOTE, ESC_QUOTE2])
604 604
605 605 def handle(self, line_info):
606 606 """Handle lines which can be auto-executed, quoting if requested."""
607 607 line = line_info.line
608 608 ifun = line_info.ifun
609 609 the_rest = line_info.the_rest
610 610 esc = line_info.esc
611 611 continue_prompt = line_info.continue_prompt
612 612 obj = line_info.ofind(self.shell)['obj']
613 613
614 614 # This should only be active for single-line input!
615 615 if continue_prompt:
616 616 return line
617 617
618 618 force_auto = isinstance(obj, IPyAutocall)
619 619
620 620 # User objects sometimes raise exceptions on attribute access other
621 621 # than AttributeError (we've seen it in the past), so it's safest to be
622 622 # ultra-conservative here and catch all.
623 623 try:
624 624 auto_rewrite = obj.rewrite
625 625 except Exception:
626 626 auto_rewrite = True
627 627
628 628 if esc == ESC_QUOTE:
629 629 # Auto-quote splitting on whitespace
630 630 newcmd = '%s("%s")' % (ifun,'", "'.join(the_rest.split()) )
631 631 elif esc == ESC_QUOTE2:
632 632 # Auto-quote whole string
633 633 newcmd = '%s("%s")' % (ifun,the_rest)
634 634 elif esc == ESC_PAREN:
635 635 newcmd = '%s(%s)' % (ifun,",".join(the_rest.split()))
636 636 else:
637 637 # Auto-paren.
638 638 if force_auto:
639 639 # Don't rewrite if it is already a call.
640 640 do_rewrite = not the_rest.startswith('(')
641 641 else:
642 642 if not the_rest:
643 643 # We only apply it to argument-less calls if the autocall
644 644 # parameter is set to 2.
645 645 do_rewrite = (self.shell.autocall >= 2)
646 646 elif the_rest.startswith('[') and hasattr(obj, '__getitem__'):
647 647 # Don't autocall in this case: item access for an object
648 648 # which is BOTH callable and implements __getitem__.
649 649 do_rewrite = False
650 650 else:
651 651 do_rewrite = True
652 652
653 653 # Figure out the rewritten command
654 654 if do_rewrite:
655 655 if the_rest.endswith(';'):
656 656 newcmd = '%s(%s);' % (ifun.rstrip(),the_rest[:-1])
657 657 else:
658 658 newcmd = '%s(%s)' % (ifun.rstrip(), the_rest)
659 659 else:
660 660 normal_handler = self.prefilter_manager.get_handler_by_name('normal')
661 661 return normal_handler.handle(line_info)
662 662
663 663 # Display the rewritten call
664 664 if auto_rewrite:
665 665 self.shell.auto_rewrite_input(newcmd)
666 666
667 667 return newcmd
668 668
669 669
670 670 class EmacsHandler(PrefilterHandler):
671 671
672 672 handler_name = Unicode('emacs')
673 673 esc_strings = List([])
674 674
675 675 def handle(self, line_info):
676 676 """Handle input lines marked by python-mode."""
677 677
678 678 # Currently, nothing is done. Later more functionality can be added
679 679 # here if needed.
680 680
681 681 # The input cache shouldn't be updated
682 682 return line_info.line
683 683
684 684
685 685 #-----------------------------------------------------------------------------
686 686 # Defaults
687 687 #-----------------------------------------------------------------------------
688 688
689 689
690 690 _default_transformers = [
691 691 ]
692 692
693 693 _default_checkers = [
694 694 EmacsChecker,
695 695 MacroChecker,
696 696 IPyAutocallChecker,
697 697 AssignmentChecker,
698 698 AutoMagicChecker,
699 699 PythonOpsChecker,
700 700 AutocallChecker
701 701 ]
702 702
703 703 _default_handlers = [
704 704 PrefilterHandler,
705 705 MacroHandler,
706 706 MagicHandler,
707 707 AutoHandler,
708 708 EmacsHandler
709 709 ]
@@ -1,494 +1,495 b''
1 1 import tokenize
2 2 import nose.tools as nt
3 3
4 4 from IPython.testing import tools as tt
5 5 from IPython.utils import py3compat
6 6 u_fmt = py3compat.u_format
7 7
8 8 from IPython.core import inputtransformer as ipt
9 9
10 10 def transform_and_reset(transformer):
11 11 transformer = transformer()
12 12 def transform(inp):
13 13 try:
14 14 return transformer.push(inp)
15 15 finally:
16 16 transformer.reset()
17 17
18 18 return transform
19 19
20 20 # Transformer tests
21 21 def transform_checker(tests, transformer, **kwargs):
22 22 """Utility to loop over test inputs"""
23 23 transformer = transformer(**kwargs)
24 24 try:
25 25 for inp, tr in tests:
26 26 if inp is None:
27 27 out = transformer.reset()
28 28 else:
29 29 out = transformer.push(inp)
30 30 nt.assert_equal(out, tr)
31 31 finally:
32 32 transformer.reset()
33 33
34 34 # Data for all the syntax tests in the form of lists of pairs of
35 35 # raw/transformed input. We store it here as a global dict so that we can use
36 36 # it both within single-function tests and also to validate the behavior of the
37 37 # larger objects
38 38
39 39 syntax = \
40 40 dict(assign_system =
41 41 [(i,py3compat.u_format(o)) for i,o in \
42 42 [(u'a =! ls', "a = get_ipython().getoutput('ls')"),
43 43 (u'b = !ls', "b = get_ipython().getoutput('ls')"),
44 44 (u'c= !ls', "c = get_ipython().getoutput('ls')"),
45 45 (u'd == !ls', u'd == !ls'), # Invalid syntax, but we leave == alone.
46 46 ('x=1', 'x=1'), # normal input is unmodified
47 47 (' ',' '), # blank lines are kept intact
48 48 # Tuple unpacking
49 49 (u"a, b = !echo 'a\\nb'", u"a, b = get_ipython().getoutput(\"echo 'a\\\\nb'\")"),
50 50 (u"a,= !echo 'a'", u"a, = get_ipython().getoutput(\"echo 'a'\")"),
51 51 (u"a, *bc = !echo 'a\\nb\\nc'", u"a, *bc = get_ipython().getoutput(\"echo 'a\\\\nb\\\\nc'\")"),
52 52 # Tuple unpacking with regular Python expressions, not our syntax.
53 53 (u"a, b = range(2)", u"a, b = range(2)"),
54 54 (u"a, = range(1)", u"a, = range(1)"),
55 55 (u"a, *bc = range(3)", u"a, *bc = range(3)"),
56 56 ]],
57 57
58 58 assign_magic =
59 59 [(i,py3compat.u_format(o)) for i,o in \
60 60 [(u'a =% who', "a = get_ipython().run_line_magic('who', '')"),
61 61 (u'b = %who', "b = get_ipython().run_line_magic('who', '')"),
62 62 (u'c= %ls', "c = get_ipython().run_line_magic('ls', '')"),
63 63 (u'd == %ls', u'd == %ls'), # Invalid syntax, but we leave == alone.
64 64 ('x=1', 'x=1'), # normal input is unmodified
65 65 (' ',' '), # blank lines are kept intact
66 66 (u"a, b = %foo", u"a, b = get_ipython().run_line_magic('foo', '')"),
67 67 ]],
68 68
69 69 classic_prompt =
70 70 [('>>> x=1', 'x=1'),
71 71 ('x=1', 'x=1'), # normal input is unmodified
72 72 (' ', ' '), # blank lines are kept intact
73 73 ],
74 74
75 75 ipy_prompt =
76 76 [('In [1]: x=1', 'x=1'),
77 77 ('x=1', 'x=1'), # normal input is unmodified
78 78 (' ',' '), # blank lines are kept intact
79 79 ],
80 80
81 81 # Tests for the escape transformer to leave normal code alone
82 82 escaped_noesc =
83 83 [ (' ', ' '),
84 84 ('x=1', 'x=1'),
85 85 ],
86 86
87 87 # System calls
88 88 escaped_shell =
89 89 [(i,py3compat.u_format(o)) for i,o in \
90 90 [ (u'!ls', "get_ipython().system('ls')"),
91 91 # Double-escape shell, this means to capture the output of the
92 92 # subprocess and return it
93 93 (u'!!ls', "get_ipython().getoutput('ls')"),
94 94 ]],
95 95
96 96 # Help/object info
97 97 escaped_help =
98 98 [(i,py3compat.u_format(o)) for i,o in \
99 99 [ (u'?', 'get_ipython().show_usage()'),
100 100 (u'?x1', "get_ipython().run_line_magic('pinfo', 'x1')"),
101 101 (u'??x2', "get_ipython().run_line_magic('pinfo2', 'x2')"),
102 102 (u'?a.*s', "get_ipython().run_line_magic('psearch', 'a.*s')"),
103 103 (u'?%hist1', "get_ipython().run_line_magic('pinfo', '%hist1')"),
104 104 (u'?%%hist2', "get_ipython().run_line_magic('pinfo', '%%hist2')"),
105 105 (u'?abc = qwe', "get_ipython().run_line_magic('pinfo', 'abc')"),
106 106 ]],
107 107
108 108 end_help =
109 109 [(i,py3compat.u_format(o)) for i,o in \
110 110 [ (u'x3?', "get_ipython().run_line_magic('pinfo', 'x3')"),
111 111 (u'x4??', "get_ipython().run_line_magic('pinfo2', 'x4')"),
112 112 (u'%hist1?', "get_ipython().run_line_magic('pinfo', '%hist1')"),
113 113 (u'%hist2??', "get_ipython().run_line_magic('pinfo2', '%hist2')"),
114 114 (u'%%hist3?', "get_ipython().run_line_magic('pinfo', '%%hist3')"),
115 115 (u'%%hist4??', "get_ipython().run_line_magic('pinfo2', '%%hist4')"),
116 (u'π.foo?', "get_ipython().run_line_magic('pinfo', 'π.foo')"),
116 117 (u'f*?', "get_ipython().run_line_magic('psearch', 'f*')"),
117 118 (u'ax.*aspe*?', "get_ipython().run_line_magic('psearch', 'ax.*aspe*')"),
118 119 (u'a = abc?', "get_ipython().set_next_input('a = abc');"
119 120 "get_ipython().run_line_magic('pinfo', 'abc')"),
120 121 (u'a = abc.qe??', "get_ipython().set_next_input('a = abc.qe');"
121 122 "get_ipython().run_line_magic('pinfo2', 'abc.qe')"),
122 123 (u'a = *.items?', "get_ipython().set_next_input('a = *.items');"
123 124 "get_ipython().run_line_magic('psearch', '*.items')"),
124 125 (u'plot(a?', "get_ipython().set_next_input('plot(a');"
125 126 "get_ipython().run_line_magic('pinfo', 'a')"),
126 127 (u'a*2 #comment?', 'a*2 #comment?'),
127 128 ]],
128 129
129 130 # Explicit magic calls
130 131 escaped_magic =
131 132 [(i,py3compat.u_format(o)) for i,o in \
132 133 [ (u'%cd', "get_ipython().run_line_magic('cd', '')"),
133 134 (u'%cd /home', "get_ipython().run_line_magic('cd', '/home')"),
134 135 # Backslashes need to be escaped.
135 136 (u'%cd C:\\User', "get_ipython().run_line_magic('cd', 'C:\\\\User')"),
136 137 (u' %magic', " get_ipython().run_line_magic('magic', '')"),
137 138 ]],
138 139
139 140 # Quoting with separate arguments
140 141 escaped_quote =
141 142 [ (',f', 'f("")'),
142 143 (',f x', 'f("x")'),
143 144 (' ,f y', ' f("y")'),
144 145 (',f a b', 'f("a", "b")'),
145 146 ],
146 147
147 148 # Quoting with single argument
148 149 escaped_quote2 =
149 150 [ (';f', 'f("")'),
150 151 (';f x', 'f("x")'),
151 152 (' ;f y', ' f("y")'),
152 153 (';f a b', 'f("a b")'),
153 154 ],
154 155
155 156 # Simply apply parens
156 157 escaped_paren =
157 158 [ ('/f', 'f()'),
158 159 ('/f x', 'f(x)'),
159 160 (' /f y', ' f(y)'),
160 161 ('/f a b', 'f(a, b)'),
161 162 ],
162 163
163 164 # Check that we transform prompts before other transforms
164 165 mixed =
165 166 [(i,py3compat.u_format(o)) for i,o in \
166 167 [ (u'In [1]: %lsmagic', "get_ipython().run_line_magic('lsmagic', '')"),
167 168 (u'>>> %lsmagic', "get_ipython().run_line_magic('lsmagic', '')"),
168 169 (u'In [2]: !ls', "get_ipython().system('ls')"),
169 170 (u'In [3]: abs?', "get_ipython().run_line_magic('pinfo', 'abs')"),
170 171 (u'In [4]: b = %who', "b = get_ipython().run_line_magic('who', '')"),
171 172 ]],
172 173 )
173 174
174 175 # multiline syntax examples. Each of these should be a list of lists, with
175 176 # each entry itself having pairs of raw/transformed input. The union (with
176 177 # '\n'.join() of the transformed inputs is what the splitter should produce
177 178 # when fed the raw lines one at a time via push.
178 179 syntax_ml = \
179 180 dict(classic_prompt =
180 181 [ [('>>> for i in range(10):','for i in range(10):'),
181 182 ('... print i',' print i'),
182 183 ('... ', ''),
183 184 ],
184 185 [('>>> a="""','a="""'),
185 186 ('... 123"""','123"""'),
186 187 ],
187 188 [('a="""','a="""'),
188 189 ('... 123','123'),
189 190 ('... 456"""','456"""'),
190 191 ],
191 192 [('a="""','a="""'),
192 193 ('>>> 123','123'),
193 194 ('... 456"""','456"""'),
194 195 ],
195 196 [('a="""','a="""'),
196 197 ('123','123'),
197 198 ('... 456"""','... 456"""'),
198 199 ],
199 200 [('....__class__','....__class__'),
200 201 ],
201 202 [('a=5', 'a=5'),
202 203 ('...', ''),
203 204 ],
204 205 [('>>> def f(x):', 'def f(x):'),
205 206 ('...', ''),
206 207 ('... return x', ' return x'),
207 208 ],
208 209 [('board = """....', 'board = """....'),
209 210 ('....', '....'),
210 211 ('...."""', '...."""'),
211 212 ],
212 213 ],
213 214
214 215 ipy_prompt =
215 216 [ [('In [24]: for i in range(10):','for i in range(10):'),
216 217 (' ....: print i',' print i'),
217 218 (' ....: ', ''),
218 219 ],
219 220 [('In [24]: for i in range(10):','for i in range(10):'),
220 221 # Qt console prompts expand with spaces, not dots
221 222 (' ...: print i',' print i'),
222 223 (' ...: ', ''),
223 224 ],
224 225 [('In [24]: for i in range(10):','for i in range(10):'),
225 226 # Sometimes whitespace preceding '...' has been removed
226 227 ('...: print i',' print i'),
227 228 ('...: ', ''),
228 229 ],
229 230 [('In [24]: for i in range(10):','for i in range(10):'),
230 231 # Space after last continuation prompt has been removed (issue #6674)
231 232 ('...: print i',' print i'),
232 233 ('...:', ''),
233 234 ],
234 235 [('In [2]: a="""','a="""'),
235 236 (' ...: 123"""','123"""'),
236 237 ],
237 238 [('a="""','a="""'),
238 239 (' ...: 123','123'),
239 240 (' ...: 456"""','456"""'),
240 241 ],
241 242 [('a="""','a="""'),
242 243 ('In [1]: 123','123'),
243 244 (' ...: 456"""','456"""'),
244 245 ],
245 246 [('a="""','a="""'),
246 247 ('123','123'),
247 248 (' ...: 456"""',' ...: 456"""'),
248 249 ],
249 250 ],
250 251
251 252 multiline_datastructure_prompt =
252 253 [ [('>>> a = [1,','a = [1,'),
253 254 ('... 2]','2]'),
254 255 ],
255 256 ],
256 257
257 258 multiline_datastructure =
258 259 [ [('b = ("%s"', None),
259 260 ('# comment', None),
260 261 ('%foo )', 'b = ("%s"\n# comment\n%foo )'),
261 262 ],
262 263 ],
263 264
264 265 multiline_string =
265 266 [ [("'''foo?", None),
266 267 ("bar'''", "'''foo?\nbar'''"),
267 268 ],
268 269 ],
269 270
270 271 leading_indent =
271 272 [ [(' print "hi"','print "hi"'),
272 273 ],
273 274 [(' for a in range(5):','for a in range(5):'),
274 275 (' a*2',' a*2'),
275 276 ],
276 277 [(' a="""','a="""'),
277 278 (' 123"""','123"""'),
278 279 ],
279 280 [('a="""','a="""'),
280 281 (' 123"""',' 123"""'),
281 282 ],
282 283 ],
283 284
284 285 cellmagic =
285 286 [ [(u'%%foo a', None),
286 287 (None, u_fmt("get_ipython().run_cell_magic('foo', 'a', '')")),
287 288 ],
288 289 [(u'%%bar 123', None),
289 290 (u'hello', None),
290 291 (None , u_fmt("get_ipython().run_cell_magic('bar', '123', 'hello')")),
291 292 ],
292 293 [(u'a=5', 'a=5'),
293 294 (u'%%cellmagic', '%%cellmagic'),
294 295 ],
295 296 ],
296 297
297 298 escaped =
298 299 [ [('%abc def \\', None),
299 300 ('ghi', u_fmt("get_ipython().run_line_magic('abc', 'def ghi')")),
300 301 ],
301 302 [('%abc def \\', None),
302 303 ('ghi\\', None),
303 304 (None, u_fmt("get_ipython().run_line_magic('abc', 'def ghi')")),
304 305 ],
305 306 ],
306 307
307 308 assign_magic =
308 309 [ [(u'a = %bc de \\', None),
309 310 (u'fg', u_fmt("a = get_ipython().run_line_magic('bc', 'de fg')")),
310 311 ],
311 312 [(u'a = %bc de \\', None),
312 313 (u'fg\\', None),
313 314 (None, u_fmt("a = get_ipython().run_line_magic('bc', 'de fg')")),
314 315 ],
315 316 ],
316 317
317 318 assign_system =
318 319 [ [(u'a = !bc de \\', None),
319 320 (u'fg', u_fmt("a = get_ipython().getoutput('bc de fg')")),
320 321 ],
321 322 [(u'a = !bc de \\', None),
322 323 (u'fg\\', None),
323 324 (None, u_fmt("a = get_ipython().getoutput('bc de fg')")),
324 325 ],
325 326 ],
326 327 )
327 328
328 329
329 330 def test_assign_system():
330 331 tt.check_pairs(transform_and_reset(ipt.assign_from_system), syntax['assign_system'])
331 332
332 333 def test_assign_magic():
333 334 tt.check_pairs(transform_and_reset(ipt.assign_from_magic), syntax['assign_magic'])
334 335
335 336 def test_classic_prompt():
336 337 tt.check_pairs(transform_and_reset(ipt.classic_prompt), syntax['classic_prompt'])
337 338 for example in syntax_ml['classic_prompt']:
338 339 transform_checker(example, ipt.classic_prompt)
339 340 for example in syntax_ml['multiline_datastructure_prompt']:
340 341 transform_checker(example, ipt.classic_prompt)
341 342
342 343 # Check that we don't transform the second line if the first is obviously
343 344 # IPython syntax
344 345 transform_checker([
345 346 (u'%foo', '%foo'),
346 347 (u'>>> bar', '>>> bar'),
347 348 ], ipt.classic_prompt)
348 349
349 350
350 351 def test_ipy_prompt():
351 352 tt.check_pairs(transform_and_reset(ipt.ipy_prompt), syntax['ipy_prompt'])
352 353 for example in syntax_ml['ipy_prompt']:
353 354 transform_checker(example, ipt.ipy_prompt)
354 355
355 356 # Check that we don't transform the second line if we're inside a cell magic
356 357 transform_checker([
357 358 (u'%%foo', '%%foo'),
358 359 (u'In [1]: bar', 'In [1]: bar'),
359 360 ], ipt.ipy_prompt)
360 361
361 362 def test_assemble_logical_lines():
362 363 tests = \
363 364 [ [(u"a = \\", None),
364 365 (u"123", u"a = 123"),
365 366 ],
366 367 [(u"a = \\", None), # Test resetting when within a multi-line string
367 368 (u"12 *\\", None),
368 369 (None, u"a = 12 *"),
369 370 ],
370 371 [(u"# foo\\", u"# foo\\"), # Comments can't be continued like this
371 372 ],
372 373 ]
373 374 for example in tests:
374 375 transform_checker(example, ipt.assemble_logical_lines)
375 376
376 377 def test_assemble_python_lines():
377 378 tests = \
378 379 [ [(u"a = '''", None),
379 380 (u"abc'''", u"a = '''\nabc'''"),
380 381 ],
381 382 [(u"a = '''", None), # Test resetting when within a multi-line string
382 383 (u"def", None),
383 384 (None, u"a = '''\ndef"),
384 385 ],
385 386 [(u"a = [1,", None),
386 387 (u"2]", u"a = [1,\n2]"),
387 388 ],
388 389 [(u"a = [1,", None), # Test resetting when within a multi-line string
389 390 (u"2,", None),
390 391 (None, u"a = [1,\n2,"),
391 392 ],
392 393 [(u"a = '''", None), # Test line continuation within a multi-line string
393 394 (u"abc\\", None),
394 395 (u"def", None),
395 396 (u"'''", u"a = '''\nabc\\\ndef\n'''"),
396 397 ],
397 398 ] + syntax_ml['multiline_datastructure']
398 399 for example in tests:
399 400 transform_checker(example, ipt.assemble_python_lines)
400 401
401 402
402 403 def test_help_end():
403 404 tt.check_pairs(transform_and_reset(ipt.help_end), syntax['end_help'])
404 405
405 406 def test_escaped_noesc():
406 407 tt.check_pairs(transform_and_reset(ipt.escaped_commands), syntax['escaped_noesc'])
407 408
408 409
409 410 def test_escaped_shell():
410 411 tt.check_pairs(transform_and_reset(ipt.escaped_commands), syntax['escaped_shell'])
411 412
412 413
413 414 def test_escaped_help():
414 415 tt.check_pairs(transform_and_reset(ipt.escaped_commands), syntax['escaped_help'])
415 416
416 417
417 418 def test_escaped_magic():
418 419 tt.check_pairs(transform_and_reset(ipt.escaped_commands), syntax['escaped_magic'])
419 420
420 421
421 422 def test_escaped_quote():
422 423 tt.check_pairs(transform_and_reset(ipt.escaped_commands), syntax['escaped_quote'])
423 424
424 425
425 426 def test_escaped_quote2():
426 427 tt.check_pairs(transform_and_reset(ipt.escaped_commands), syntax['escaped_quote2'])
427 428
428 429
429 430 def test_escaped_paren():
430 431 tt.check_pairs(transform_and_reset(ipt.escaped_commands), syntax['escaped_paren'])
431 432
432 433
433 434 def test_cellmagic():
434 435 for example in syntax_ml['cellmagic']:
435 436 transform_checker(example, ipt.cellmagic)
436 437
437 438 line_example = [(u'%%bar 123', None),
438 439 (u'hello', None),
439 440 (u'' , u_fmt("get_ipython().run_cell_magic('bar', '123', 'hello')")),
440 441 ]
441 442 transform_checker(line_example, ipt.cellmagic, end_on_blank_line=True)
442 443
443 444 def test_has_comment():
444 445 tests = [('text', False),
445 446 ('text #comment', True),
446 447 ('text #comment\n', True),
447 448 ('#comment', True),
448 449 ('#comment\n', True),
449 450 ('a = "#string"', False),
450 451 ('a = "#string" # comment', True),
451 452 ('a #comment not "string"', True),
452 453 ]
453 454 tt.check_pairs(ipt.has_comment, tests)
454 455
455 456 @ipt.TokenInputTransformer.wrap
456 457 def decistmt(tokens):
457 458 """Substitute Decimals for floats in a string of statements.
458 459
459 460 Based on an example from the tokenize module docs.
460 461 """
461 462 result = []
462 463 for toknum, tokval, _, _, _ in tokens:
463 464 if toknum == tokenize.NUMBER and '.' in tokval: # replace NUMBER tokens
464 465 for newtok in [
465 466 (tokenize.NAME, 'Decimal'),
466 467 (tokenize.OP, '('),
467 468 (tokenize.STRING, repr(tokval)),
468 469 (tokenize.OP, ')')
469 470 ]:
470 471 yield newtok
471 472 else:
472 473 yield (toknum, tokval)
473 474
474 475
475 476
476 477 def test_token_input_transformer():
477 478 tests = [(u'1.2', u_fmt(u"Decimal ('1.2')")),
478 479 (u'"1.2"', u'"1.2"'),
479 480 ]
480 481 tt.check_pairs(transform_and_reset(decistmt), tests)
481 482 ml_tests = \
482 483 [ [(u"a = 1.2; b = '''x", None),
483 484 (u"y'''", u_fmt(u"a =Decimal ('1.2');b ='''x\ny'''")),
484 485 ],
485 486 [(u"a = [1.2,", None),
486 487 (u"3]", u_fmt(u"a =[Decimal ('1.2'),\n3 ]")),
487 488 ],
488 489 [(u"a = '''foo", None), # Test resetting when within a multi-line string
489 490 (u"bar", None),
490 491 (None, u"a = '''foo\nbar"),
491 492 ],
492 493 ]
493 494 for example in ml_tests:
494 495 transform_checker(example, decistmt)
@@ -1,284 +1,292 b''
1 1 """Tests for the token-based transformers in IPython.core.inputtransformer2
2 2
3 3 Line-based transformers are the simpler ones; token-based transformers are
4 4 more complex. See test_inputtransformer2_line for tests for line-based
5 5 transformations.
6 6 """
7 7 import nose.tools as nt
8 8 import string
9 9
10 10 from IPython.core import inputtransformer2 as ipt2
11 11 from IPython.core.inputtransformer2 import make_tokens_by_line, _find_assign_op
12 12
13 13 from textwrap import dedent
14 14
15 15 MULTILINE_MAGIC = ("""\
16 16 a = f()
17 17 %foo \\
18 18 bar
19 19 g()
20 20 """.splitlines(keepends=True), (2, 0), """\
21 21 a = f()
22 22 get_ipython().run_line_magic('foo', ' bar')
23 23 g()
24 24 """.splitlines(keepends=True))
25 25
26 26 INDENTED_MAGIC = ("""\
27 27 for a in range(5):
28 28 %ls
29 29 """.splitlines(keepends=True), (2, 4), """\
30 30 for a in range(5):
31 31 get_ipython().run_line_magic('ls', '')
32 32 """.splitlines(keepends=True))
33 33
34 34 MULTILINE_MAGIC_ASSIGN = ("""\
35 35 a = f()
36 36 b = %foo \\
37 37 bar
38 38 g()
39 39 """.splitlines(keepends=True), (2, 4), """\
40 40 a = f()
41 41 b = get_ipython().run_line_magic('foo', ' bar')
42 42 g()
43 43 """.splitlines(keepends=True))
44 44
45 45 MULTILINE_SYSTEM_ASSIGN = ("""\
46 46 a = f()
47 47 b = !foo \\
48 48 bar
49 49 g()
50 50 """.splitlines(keepends=True), (2, 4), """\
51 51 a = f()
52 52 b = get_ipython().getoutput('foo bar')
53 53 g()
54 54 """.splitlines(keepends=True))
55 55
56 56 #####
57 57
58 58 MULTILINE_SYSTEM_ASSIGN_AFTER_DEDENT = ("""\
59 59 def test():
60 60 for i in range(1):
61 61 print(i)
62 62 res =! ls
63 63 """.splitlines(keepends=True), (4, 7), '''\
64 64 def test():
65 65 for i in range(1):
66 66 print(i)
67 67 res =get_ipython().getoutput(\' ls\')
68 68 '''.splitlines(keepends=True))
69 69
70 70 ######
71 71
72 72 AUTOCALL_QUOTE = (
73 73 [",f 1 2 3\n"], (1, 0),
74 74 ['f("1", "2", "3")\n']
75 75 )
76 76
77 77 AUTOCALL_QUOTE2 = (
78 78 [";f 1 2 3\n"], (1, 0),
79 79 ['f("1 2 3")\n']
80 80 )
81 81
82 82 AUTOCALL_PAREN = (
83 83 ["/f 1 2 3\n"], (1, 0),
84 84 ['f(1, 2, 3)\n']
85 85 )
86 86
87 87 SIMPLE_HELP = (
88 88 ["foo?\n"], (1, 0),
89 89 ["get_ipython().run_line_magic('pinfo', 'foo')\n"]
90 90 )
91 91
92 92 DETAILED_HELP = (
93 93 ["foo??\n"], (1, 0),
94 94 ["get_ipython().run_line_magic('pinfo2', 'foo')\n"]
95 95 )
96 96
97 97 MAGIC_HELP = (
98 98 ["%foo?\n"], (1, 0),
99 99 ["get_ipython().run_line_magic('pinfo', '%foo')\n"]
100 100 )
101 101
102 102 HELP_IN_EXPR = (
103 103 ["a = b + c?\n"], (1, 0),
104 104 ["get_ipython().set_next_input('a = b + c');"
105 105 "get_ipython().run_line_magic('pinfo', 'c')\n"]
106 106 )
107 107
108 108 HELP_CONTINUED_LINE = ("""\
109 109 a = \\
110 110 zip?
111 111 """.splitlines(keepends=True), (1, 0),
112 112 [r"get_ipython().set_next_input('a = \\\nzip');get_ipython().run_line_magic('pinfo', 'zip')" + "\n"]
113 113 )
114 114
115 115 HELP_MULTILINE = ("""\
116 116 (a,
117 117 b) = zip?
118 118 """.splitlines(keepends=True), (1, 0),
119 119 [r"get_ipython().set_next_input('(a,\nb) = zip');get_ipython().run_line_magic('pinfo', 'zip')" + "\n"]
120 120 )
121 121
122 HELP_UNICODE = (
123 ["π.foo?\n"], (1, 0),
124 ["get_ipython().run_line_magic('pinfo', 'π.foo')\n"]
125 )
126
122 127
123 128 def null_cleanup_transformer(lines):
124 129 """
125 130 A cleanup transform that returns an empty list.
126 131 """
127 132 return []
128 133
129 134 def check_make_token_by_line_never_ends_empty():
130 135 """
131 136 Check that not sequence of single or double characters ends up leading to en empty list of tokens
132 137 """
133 138 from string import printable
134 139 for c in printable:
135 140 nt.assert_not_equal(make_tokens_by_line(c)[-1], [])
136 141 for k in printable:
137 142 nt.assert_not_equal(make_tokens_by_line(c+k)[-1], [])
138 143
139 144 def check_find(transformer, case, match=True):
140 145 sample, expected_start, _ = case
141 146 tbl = make_tokens_by_line(sample)
142 147 res = transformer.find(tbl)
143 148 if match:
144 149 # start_line is stored 0-indexed, expected values are 1-indexed
145 150 nt.assert_equal((res.start_line+1, res.start_col), expected_start)
146 151 return res
147 152 else:
148 153 nt.assert_is(res, None)
149 154
150 155 def check_transform(transformer_cls, case):
151 156 lines, start, expected = case
152 157 transformer = transformer_cls(start)
153 158 nt.assert_equal(transformer.transform(lines), expected)
154 159
155 160 def test_continued_line():
156 161 lines = MULTILINE_MAGIC_ASSIGN[0]
157 162 nt.assert_equal(ipt2.find_end_of_continued_line(lines, 1), 2)
158 163
159 164 nt.assert_equal(ipt2.assemble_continued_line(lines, (1, 5), 2), "foo bar")
160 165
161 166 def test_find_assign_magic():
162 167 check_find(ipt2.MagicAssign, MULTILINE_MAGIC_ASSIGN)
163 168 check_find(ipt2.MagicAssign, MULTILINE_SYSTEM_ASSIGN, match=False)
164 169 check_find(ipt2.MagicAssign, MULTILINE_SYSTEM_ASSIGN_AFTER_DEDENT, match=False)
165 170
166 171 def test_transform_assign_magic():
167 172 check_transform(ipt2.MagicAssign, MULTILINE_MAGIC_ASSIGN)
168 173
169 174 def test_find_assign_system():
170 175 check_find(ipt2.SystemAssign, MULTILINE_SYSTEM_ASSIGN)
171 176 check_find(ipt2.SystemAssign, MULTILINE_SYSTEM_ASSIGN_AFTER_DEDENT)
172 177 check_find(ipt2.SystemAssign, (["a = !ls\n"], (1, 5), None))
173 178 check_find(ipt2.SystemAssign, (["a=!ls\n"], (1, 2), None))
174 179 check_find(ipt2.SystemAssign, MULTILINE_MAGIC_ASSIGN, match=False)
175 180
176 181 def test_transform_assign_system():
177 182 check_transform(ipt2.SystemAssign, MULTILINE_SYSTEM_ASSIGN)
178 183 check_transform(ipt2.SystemAssign, MULTILINE_SYSTEM_ASSIGN_AFTER_DEDENT)
179 184
180 185 def test_find_magic_escape():
181 186 check_find(ipt2.EscapedCommand, MULTILINE_MAGIC)
182 187 check_find(ipt2.EscapedCommand, INDENTED_MAGIC)
183 188 check_find(ipt2.EscapedCommand, MULTILINE_MAGIC_ASSIGN, match=False)
184 189
185 190 def test_transform_magic_escape():
186 191 check_transform(ipt2.EscapedCommand, MULTILINE_MAGIC)
187 192 check_transform(ipt2.EscapedCommand, INDENTED_MAGIC)
188 193
189 194 def test_find_autocalls():
190 195 for case in [AUTOCALL_QUOTE, AUTOCALL_QUOTE2, AUTOCALL_PAREN]:
191 196 print("Testing %r" % case[0])
192 197 check_find(ipt2.EscapedCommand, case)
193 198
194 199 def test_transform_autocall():
195 200 for case in [AUTOCALL_QUOTE, AUTOCALL_QUOTE2, AUTOCALL_PAREN]:
196 201 print("Testing %r" % case[0])
197 202 check_transform(ipt2.EscapedCommand, case)
198 203
199 204 def test_find_help():
200 205 for case in [SIMPLE_HELP, DETAILED_HELP, MAGIC_HELP, HELP_IN_EXPR]:
201 206 check_find(ipt2.HelpEnd, case)
202 207
203 208 tf = check_find(ipt2.HelpEnd, HELP_CONTINUED_LINE)
204 209 nt.assert_equal(tf.q_line, 1)
205 210 nt.assert_equal(tf.q_col, 3)
206 211
207 212 tf = check_find(ipt2.HelpEnd, HELP_MULTILINE)
208 213 nt.assert_equal(tf.q_line, 1)
209 214 nt.assert_equal(tf.q_col, 8)
210 215
211 216 # ? in a comment does not trigger help
212 217 check_find(ipt2.HelpEnd, (["foo # bar?\n"], None, None), match=False)
213 218 # Nor in a string
214 219 check_find(ipt2.HelpEnd, (["foo = '''bar?\n"], None, None), match=False)
215 220
216 221 def test_transform_help():
217 222 tf = ipt2.HelpEnd((1, 0), (1, 9))
218 223 nt.assert_equal(tf.transform(HELP_IN_EXPR[0]), HELP_IN_EXPR[2])
219 224
220 225 tf = ipt2.HelpEnd((1, 0), (2, 3))
221 226 nt.assert_equal(tf.transform(HELP_CONTINUED_LINE[0]), HELP_CONTINUED_LINE[2])
222 227
223 228 tf = ipt2.HelpEnd((1, 0), (2, 8))
224 229 nt.assert_equal(tf.transform(HELP_MULTILINE[0]), HELP_MULTILINE[2])
225 230
231 tf = ipt2.HelpEnd((1, 0), (1, 0))
232 nt.assert_equal(tf.transform(HELP_UNICODE[0]), HELP_UNICODE[2])
233
226 234 def test_find_assign_op_dedent():
227 235 """
228 236 be careful that empty token like dedent are not counted as parens
229 237 """
230 238 class Tk:
231 239 def __init__(self, s):
232 240 self.string = s
233 241
234 242 nt.assert_equal(_find_assign_op([Tk(s) for s in ('','a','=','b')]), 2)
235 243 nt.assert_equal(_find_assign_op([Tk(s) for s in ('','(', 'a','=','b', ')', '=' ,'5')]), 6)
236 244
237 245 def test_check_complete():
238 246 cc = ipt2.TransformerManager().check_complete
239 247 nt.assert_equal(cc("a = 1"), ('complete', None))
240 248 nt.assert_equal(cc("for a in range(5):"), ('incomplete', 4))
241 249 nt.assert_equal(cc("for a in range(5):\n if a > 0:"), ('incomplete', 8))
242 250 nt.assert_equal(cc("raise = 2"), ('invalid', None))
243 251 nt.assert_equal(cc("a = [1,\n2,"), ('incomplete', 0))
244 252 nt.assert_equal(cc(")"), ('incomplete', 0))
245 253 nt.assert_equal(cc("\\\r\n"), ('incomplete', 0))
246 254 nt.assert_equal(cc("a = '''\n hi"), ('incomplete', 3))
247 255 nt.assert_equal(cc("def a():\n x=1\n global x"), ('invalid', None))
248 256 nt.assert_equal(cc("a \\ "), ('invalid', None)) # Nothing allowed after backslash
249 257 nt.assert_equal(cc("1\\\n+2"), ('complete', None))
250 258 nt.assert_equal(cc("exit"), ('complete', None))
251 259
252 260 example = dedent("""
253 261 if True:
254 262 a=1""" )
255 263
256 264 nt.assert_equal(cc(example), ('incomplete', 4))
257 265 nt.assert_equal(cc(example+'\n'), ('complete', None))
258 266 nt.assert_equal(cc(example+'\n '), ('complete', None))
259 267
260 268 # no need to loop on all the letters/numbers.
261 269 short = '12abAB'+string.printable[62:]
262 270 for c in short:
263 271 # test does not raise:
264 272 cc(c)
265 273 for k in short:
266 274 cc(c+k)
267 275
268 276 nt.assert_equal(cc("def f():\n x=0\n \\\n "), ('incomplete', 2))
269 277
270 278 def test_check_complete_II():
271 279 """
272 280 Test that multiple line strings are properly handled.
273 281
274 282 Separate test function for convenience
275 283
276 284 """
277 285 cc = ipt2.TransformerManager().check_complete
278 286 nt.assert_equal(cc('''def foo():\n """'''), ('incomplete', 4))
279 287
280 288
281 289 def test_null_cleanup_transformer():
282 290 manager = ipt2.TransformerManager()
283 291 manager.cleanup_transforms.insert(0, null_cleanup_transformer)
284 292 nt.assert_is(manager.transform_cell(""), "")
@@ -1,1246 +1,1248 b''
1 1 # -*- coding: utf-8 -*-
2 2 """Tests for various magic functions.
3 3
4 4 Needs to be run by nose (to make ipython session available).
5 5 """
6 6
7 7 import io
8 8 import os
9 9 import re
10 10 import sys
11 11 import warnings
12 12 from textwrap import dedent
13 13 from unittest import TestCase
14 14 from unittest import mock
15 15 from importlib import invalidate_caches
16 16 from io import StringIO
17 17
18 18 import nose.tools as nt
19 19
20 20 import shlex
21 21
22 22 from IPython import get_ipython
23 23 from IPython.core import magic
24 24 from IPython.core.error import UsageError
25 25 from IPython.core.magic import (Magics, magics_class, line_magic,
26 26 cell_magic,
27 27 register_line_magic, register_cell_magic)
28 28 from IPython.core.magics import execution, script, code, logging, osm
29 29 from IPython.testing import decorators as dec
30 30 from IPython.testing import tools as tt
31 31 from IPython.utils.io import capture_output
32 32 from IPython.utils.tempdir import (TemporaryDirectory,
33 33 TemporaryWorkingDirectory)
34 34 from IPython.utils.process import find_cmd
35 35
36 36
37 37 @magic.magics_class
38 38 class DummyMagics(magic.Magics): pass
39 39
40 40 def test_extract_code_ranges():
41 41 instr = "1 3 5-6 7-9 10:15 17: :10 10- -13 :"
42 42 expected = [(0, 1),
43 43 (2, 3),
44 44 (4, 6),
45 45 (6, 9),
46 46 (9, 14),
47 47 (16, None),
48 48 (None, 9),
49 49 (9, None),
50 50 (None, 13),
51 51 (None, None)]
52 52 actual = list(code.extract_code_ranges(instr))
53 53 nt.assert_equal(actual, expected)
54 54
55 55 def test_extract_symbols():
56 56 source = """import foo\na = 10\ndef b():\n return 42\n\n\nclass A: pass\n\n\n"""
57 57 symbols_args = ["a", "b", "A", "A,b", "A,a", "z"]
58 58 expected = [([], ['a']),
59 59 (["def b():\n return 42\n"], []),
60 60 (["class A: pass\n"], []),
61 61 (["class A: pass\n", "def b():\n return 42\n"], []),
62 62 (["class A: pass\n"], ['a']),
63 63 ([], ['z'])]
64 64 for symbols, exp in zip(symbols_args, expected):
65 65 nt.assert_equal(code.extract_symbols(source, symbols), exp)
66 66
67 67
68 68 def test_extract_symbols_raises_exception_with_non_python_code():
69 69 source = ("=begin A Ruby program :)=end\n"
70 70 "def hello\n"
71 71 "puts 'Hello world'\n"
72 72 "end")
73 73 with nt.assert_raises(SyntaxError):
74 74 code.extract_symbols(source, "hello")
75 75
76 76
77 77 def test_magic_not_found():
78 78 # magic not found raises UsageError
79 79 with nt.assert_raises(UsageError):
80 80 _ip.magic('doesntexist')
81 81
82 82 # ensure result isn't success when a magic isn't found
83 83 result = _ip.run_cell('%doesntexist')
84 84 assert isinstance(result.error_in_exec, UsageError)
85 85
86 86
87 87 def test_cell_magic_not_found():
88 88 # magic not found raises UsageError
89 89 with nt.assert_raises(UsageError):
90 90 _ip.run_cell_magic('doesntexist', 'line', 'cell')
91 91
92 92 # ensure result isn't success when a magic isn't found
93 93 result = _ip.run_cell('%%doesntexist')
94 94 assert isinstance(result.error_in_exec, UsageError)
95 95
96 96
97 97 def test_magic_error_status():
98 98 def fail(shell):
99 99 1/0
100 100 _ip.register_magic_function(fail)
101 101 result = _ip.run_cell('%fail')
102 102 assert isinstance(result.error_in_exec, ZeroDivisionError)
103 103
104 104
105 105 def test_config():
106 106 """ test that config magic does not raise
107 107 can happen if Configurable init is moved too early into
108 108 Magics.__init__ as then a Config object will be registered as a
109 109 magic.
110 110 """
111 111 ## should not raise.
112 112 _ip.magic('config')
113 113
114 114 def test_config_available_configs():
115 115 """ test that config magic prints available configs in unique and
116 116 sorted order. """
117 117 with capture_output() as captured:
118 118 _ip.magic('config')
119 119
120 120 stdout = captured.stdout
121 121 config_classes = stdout.strip().split('\n')[1:]
122 122 nt.assert_list_equal(config_classes, sorted(set(config_classes)))
123 123
124 124 def test_config_print_class():
125 125 """ test that config with a classname prints the class's options. """
126 126 with capture_output() as captured:
127 127 _ip.magic('config TerminalInteractiveShell')
128 128
129 129 stdout = captured.stdout
130 130 if not re.match("TerminalInteractiveShell.* options", stdout.splitlines()[0]):
131 131 print(stdout)
132 132 raise AssertionError("1st line of stdout not like "
133 133 "'TerminalInteractiveShell.* options'")
134 134
135 135 def test_rehashx():
136 136 # clear up everything
137 137 _ip.alias_manager.clear_aliases()
138 138 del _ip.db['syscmdlist']
139 139
140 140 _ip.magic('rehashx')
141 141 # Practically ALL ipython development systems will have more than 10 aliases
142 142
143 143 nt.assert_true(len(_ip.alias_manager.aliases) > 10)
144 144 for name, cmd in _ip.alias_manager.aliases:
145 145 # we must strip dots from alias names
146 146 nt.assert_not_in('.', name)
147 147
148 148 # rehashx must fill up syscmdlist
149 149 scoms = _ip.db['syscmdlist']
150 150 nt.assert_true(len(scoms) > 10)
151 151
152 152
153 153
154 154 def test_magic_parse_options():
155 155 """Test that we don't mangle paths when parsing magic options."""
156 156 ip = get_ipython()
157 157 path = 'c:\\x'
158 158 m = DummyMagics(ip)
159 159 opts = m.parse_options('-f %s' % path,'f:')[0]
160 160 # argv splitting is os-dependent
161 161 if os.name == 'posix':
162 162 expected = 'c:x'
163 163 else:
164 164 expected = path
165 165 nt.assert_equal(opts['f'], expected)
166 166
167 167 def test_magic_parse_long_options():
168 168 """Magic.parse_options can handle --foo=bar long options"""
169 169 ip = get_ipython()
170 170 m = DummyMagics(ip)
171 171 opts, _ = m.parse_options('--foo --bar=bubble', 'a', 'foo', 'bar=')
172 172 nt.assert_in('foo', opts)
173 173 nt.assert_in('bar', opts)
174 174 nt.assert_equal(opts['bar'], "bubble")
175 175
176 176
177 177 def doctest_hist_f():
178 178 """Test %hist -f with temporary filename.
179 179
180 180 In [9]: import tempfile
181 181
182 182 In [10]: tfile = tempfile.mktemp('.py','tmp-ipython-')
183 183
184 184 In [11]: %hist -nl -f $tfile 3
185 185
186 186 In [13]: import os; os.unlink(tfile)
187 187 """
188 188
189 189
190 190 def doctest_hist_op():
191 191 """Test %hist -op
192 192
193 193 In [1]: class b(float):
194 194 ...: pass
195 195 ...:
196 196
197 197 In [2]: class s(object):
198 198 ...: def __str__(self):
199 199 ...: return 's'
200 200 ...:
201 201
202 202 In [3]:
203 203
204 204 In [4]: class r(b):
205 205 ...: def __repr__(self):
206 206 ...: return 'r'
207 207 ...:
208 208
209 209 In [5]: class sr(s,r): pass
210 210 ...:
211 211
212 212 In [6]:
213 213
214 214 In [7]: bb=b()
215 215
216 216 In [8]: ss=s()
217 217
218 218 In [9]: rr=r()
219 219
220 220 In [10]: ssrr=sr()
221 221
222 222 In [11]: 4.5
223 223 Out[11]: 4.5
224 224
225 225 In [12]: str(ss)
226 226 Out[12]: 's'
227 227
228 228 In [13]:
229 229
230 230 In [14]: %hist -op
231 231 >>> class b:
232 232 ... pass
233 233 ...
234 234 >>> class s(b):
235 235 ... def __str__(self):
236 236 ... return 's'
237 237 ...
238 238 >>>
239 239 >>> class r(b):
240 240 ... def __repr__(self):
241 241 ... return 'r'
242 242 ...
243 243 >>> class sr(s,r): pass
244 244 >>>
245 245 >>> bb=b()
246 246 >>> ss=s()
247 247 >>> rr=r()
248 248 >>> ssrr=sr()
249 249 >>> 4.5
250 250 4.5
251 251 >>> str(ss)
252 252 's'
253 253 >>>
254 254 """
255 255
256 256 def test_hist_pof():
257 257 ip = get_ipython()
258 258 ip.run_cell(u"1+2", store_history=True)
259 259 #raise Exception(ip.history_manager.session_number)
260 260 #raise Exception(list(ip.history_manager._get_range_session()))
261 261 with TemporaryDirectory() as td:
262 262 tf = os.path.join(td, 'hist.py')
263 263 ip.run_line_magic('history', '-pof %s' % tf)
264 264 assert os.path.isfile(tf)
265 265
266 266
267 267 def test_macro():
268 268 ip = get_ipython()
269 269 ip.history_manager.reset() # Clear any existing history.
270 270 cmds = ["a=1", "def b():\n return a**2", "print(a,b())"]
271 271 for i, cmd in enumerate(cmds, start=1):
272 272 ip.history_manager.store_inputs(i, cmd)
273 273 ip.magic("macro test 1-3")
274 274 nt.assert_equal(ip.user_ns["test"].value, "\n".join(cmds)+"\n")
275 275
276 276 # List macros
277 277 nt.assert_in("test", ip.magic("macro"))
278 278
279 279
280 280 def test_macro_run():
281 281 """Test that we can run a multi-line macro successfully."""
282 282 ip = get_ipython()
283 283 ip.history_manager.reset()
284 284 cmds = ["a=10", "a+=1", "print(a)", "%macro test 2-3"]
285 285 for cmd in cmds:
286 286 ip.run_cell(cmd, store_history=True)
287 287 nt.assert_equal(ip.user_ns["test"].value, "a+=1\nprint(a)\n")
288 288 with tt.AssertPrints("12"):
289 289 ip.run_cell("test")
290 290 with tt.AssertPrints("13"):
291 291 ip.run_cell("test")
292 292
293 293
294 294 def test_magic_magic():
295 295 """Test %magic"""
296 296 ip = get_ipython()
297 297 with capture_output() as captured:
298 298 ip.magic("magic")
299 299
300 300 stdout = captured.stdout
301 301 nt.assert_in('%magic', stdout)
302 302 nt.assert_in('IPython', stdout)
303 303 nt.assert_in('Available', stdout)
304 304
305 305
306 306 @dec.skipif_not_numpy
307 307 def test_numpy_reset_array_undec():
308 308 "Test '%reset array' functionality"
309 309 _ip.ex('import numpy as np')
310 310 _ip.ex('a = np.empty(2)')
311 311 nt.assert_in('a', _ip.user_ns)
312 312 _ip.magic('reset -f array')
313 313 nt.assert_not_in('a', _ip.user_ns)
314 314
315 315 def test_reset_out():
316 316 "Test '%reset out' magic"
317 317 _ip.run_cell("parrot = 'dead'", store_history=True)
318 318 # test '%reset -f out', make an Out prompt
319 319 _ip.run_cell("parrot", store_history=True)
320 320 nt.assert_true('dead' in [_ip.user_ns[x] for x in ('_','__','___')])
321 321 _ip.magic('reset -f out')
322 322 nt.assert_false('dead' in [_ip.user_ns[x] for x in ('_','__','___')])
323 323 nt.assert_equal(len(_ip.user_ns['Out']), 0)
324 324
325 325 def test_reset_in():
326 326 "Test '%reset in' magic"
327 327 # test '%reset -f in'
328 328 _ip.run_cell("parrot", store_history=True)
329 329 nt.assert_true('parrot' in [_ip.user_ns[x] for x in ('_i','_ii','_iii')])
330 330 _ip.magic('%reset -f in')
331 331 nt.assert_false('parrot' in [_ip.user_ns[x] for x in ('_i','_ii','_iii')])
332 332 nt.assert_equal(len(set(_ip.user_ns['In'])), 1)
333 333
334 334 def test_reset_dhist():
335 335 "Test '%reset dhist' magic"
336 336 _ip.run_cell("tmp = [d for d in _dh]") # copy before clearing
337 337 _ip.magic('cd ' + os.path.dirname(nt.__file__))
338 338 _ip.magic('cd -')
339 339 nt.assert_true(len(_ip.user_ns['_dh']) > 0)
340 340 _ip.magic('reset -f dhist')
341 341 nt.assert_equal(len(_ip.user_ns['_dh']), 0)
342 342 _ip.run_cell("_dh = [d for d in tmp]") #restore
343 343
344 344 def test_reset_in_length():
345 345 "Test that '%reset in' preserves In[] length"
346 346 _ip.run_cell("print 'foo'")
347 347 _ip.run_cell("reset -f in")
348 348 nt.assert_equal(len(_ip.user_ns['In']), _ip.displayhook.prompt_count+1)
349 349
350 350 class TestResetErrors(TestCase):
351 351
352 352 def test_reset_redefine(self):
353 353
354 354 @magics_class
355 355 class KernelMagics(Magics):
356 356 @line_magic
357 357 def less(self, shell): pass
358 358
359 359 _ip.register_magics(KernelMagics)
360 360
361 361 with self.assertLogs() as cm:
362 362 # hack, we want to just capture logs, but assertLogs fails if not
363 363 # logs get produce.
364 364 # so log one things we ignore.
365 365 import logging as log_mod
366 366 log = log_mod.getLogger()
367 367 log.info('Nothing')
368 368 # end hack.
369 369 _ip.run_cell("reset -f")
370 370
371 371 assert len(cm.output) == 1
372 372 for out in cm.output:
373 373 assert "Invalid alias" not in out
374 374
375 375 def test_tb_syntaxerror():
376 376 """test %tb after a SyntaxError"""
377 377 ip = get_ipython()
378 378 ip.run_cell("for")
379 379
380 380 # trap and validate stdout
381 381 save_stdout = sys.stdout
382 382 try:
383 383 sys.stdout = StringIO()
384 384 ip.run_cell("%tb")
385 385 out = sys.stdout.getvalue()
386 386 finally:
387 387 sys.stdout = save_stdout
388 388 # trim output, and only check the last line
389 389 last_line = out.rstrip().splitlines()[-1].strip()
390 390 nt.assert_equal(last_line, "SyntaxError: invalid syntax")
391 391
392 392
393 393 def test_time():
394 394 ip = get_ipython()
395 395
396 396 with tt.AssertPrints("Wall time: "):
397 397 ip.run_cell("%time None")
398 398
399 399 ip.run_cell("def f(kmjy):\n"
400 400 " %time print (2*kmjy)")
401 401
402 402 with tt.AssertPrints("Wall time: "):
403 403 with tt.AssertPrints("hihi", suppress=False):
404 404 ip.run_cell("f('hi')")
405 405
406 406 def test_time_last_not_expression():
407 407 ip.run_cell("%%time\n"
408 408 "var_1 = 1\n"
409 409 "var_2 = 2\n")
410 410 assert ip.user_ns['var_1'] == 1
411 411 del ip.user_ns['var_1']
412 412 assert ip.user_ns['var_2'] == 2
413 413 del ip.user_ns['var_2']
414 414
415 415
416 416 @dec.skip_win32
417 417 def test_time2():
418 418 ip = get_ipython()
419 419
420 420 with tt.AssertPrints("CPU times: user "):
421 421 ip.run_cell("%time None")
422 422
423 423 def test_time3():
424 424 """Erroneous magic function calls, issue gh-3334"""
425 425 ip = get_ipython()
426 426 ip.user_ns.pop('run', None)
427 427
428 428 with tt.AssertNotPrints("not found", channel='stderr'):
429 429 ip.run_cell("%%time\n"
430 430 "run = 0\n"
431 431 "run += 1")
432 432
433 433 def test_multiline_time():
434 434 """Make sure last statement from time return a value."""
435 435 ip = get_ipython()
436 436 ip.user_ns.pop('run', None)
437 437
438 438 ip.run_cell(dedent("""\
439 439 %%time
440 440 a = "ho"
441 441 b = "hey"
442 442 a+b
443 443 """))
444 444 nt.assert_equal(ip.user_ns_hidden['_'], 'hohey')
445 445
446 446 def test_time_local_ns():
447 447 """
448 448 Test that local_ns is actually global_ns when running a cell magic
449 449 """
450 450 ip = get_ipython()
451 451 ip.run_cell("%%time\n"
452 452 "myvar = 1")
453 453 nt.assert_equal(ip.user_ns['myvar'], 1)
454 454 del ip.user_ns['myvar']
455 455
456 456 def test_doctest_mode():
457 457 "Toggle doctest_mode twice, it should be a no-op and run without error"
458 458 _ip.magic('doctest_mode')
459 459 _ip.magic('doctest_mode')
460 460
461 461
462 462 def test_parse_options():
463 463 """Tests for basic options parsing in magics."""
464 464 # These are only the most minimal of tests, more should be added later. At
465 465 # the very least we check that basic text/unicode calls work OK.
466 466 m = DummyMagics(_ip)
467 467 nt.assert_equal(m.parse_options('foo', '')[1], 'foo')
468 468 nt.assert_equal(m.parse_options(u'foo', '')[1], u'foo')
469 469
470 470
471 471 def test_dirops():
472 472 """Test various directory handling operations."""
473 473 # curpath = lambda :os.path.splitdrive(os.getcwd())[1].replace('\\','/')
474 474 curpath = os.getcwd
475 475 startdir = os.getcwd()
476 476 ipdir = os.path.realpath(_ip.ipython_dir)
477 477 try:
478 478 _ip.magic('cd "%s"' % ipdir)
479 479 nt.assert_equal(curpath(), ipdir)
480 480 _ip.magic('cd -')
481 481 nt.assert_equal(curpath(), startdir)
482 482 _ip.magic('pushd "%s"' % ipdir)
483 483 nt.assert_equal(curpath(), ipdir)
484 484 _ip.magic('popd')
485 485 nt.assert_equal(curpath(), startdir)
486 486 finally:
487 487 os.chdir(startdir)
488 488
489 489
490 490 def test_cd_force_quiet():
491 491 """Test OSMagics.cd_force_quiet option"""
492 492 _ip.config.OSMagics.cd_force_quiet = True
493 493 osmagics = osm.OSMagics(shell=_ip)
494 494
495 495 startdir = os.getcwd()
496 496 ipdir = os.path.realpath(_ip.ipython_dir)
497 497
498 498 try:
499 499 with tt.AssertNotPrints(ipdir):
500 500 osmagics.cd('"%s"' % ipdir)
501 501 with tt.AssertNotPrints(startdir):
502 502 osmagics.cd('-')
503 503 finally:
504 504 os.chdir(startdir)
505 505
506 506
507 507 def test_xmode():
508 508 # Calling xmode three times should be a no-op
509 509 xmode = _ip.InteractiveTB.mode
510 510 for i in range(4):
511 511 _ip.magic("xmode")
512 512 nt.assert_equal(_ip.InteractiveTB.mode, xmode)
513 513
514 514 def test_reset_hard():
515 515 monitor = []
516 516 class A(object):
517 517 def __del__(self):
518 518 monitor.append(1)
519 519 def __repr__(self):
520 520 return "<A instance>"
521 521
522 522 _ip.user_ns["a"] = A()
523 523 _ip.run_cell("a")
524 524
525 525 nt.assert_equal(monitor, [])
526 526 _ip.magic("reset -f")
527 527 nt.assert_equal(monitor, [1])
528 528
529 529 class TestXdel(tt.TempFileMixin):
530 530 def test_xdel(self):
531 531 """Test that references from %run are cleared by xdel."""
532 532 src = ("class A(object):\n"
533 533 " monitor = []\n"
534 534 " def __del__(self):\n"
535 535 " self.monitor.append(1)\n"
536 536 "a = A()\n")
537 537 self.mktmp(src)
538 538 # %run creates some hidden references...
539 539 _ip.magic("run %s" % self.fname)
540 540 # ... as does the displayhook.
541 541 _ip.run_cell("a")
542 542
543 543 monitor = _ip.user_ns["A"].monitor
544 544 nt.assert_equal(monitor, [])
545 545
546 546 _ip.magic("xdel a")
547 547
548 548 # Check that a's __del__ method has been called.
549 549 nt.assert_equal(monitor, [1])
550 550
551 551 def doctest_who():
552 552 """doctest for %who
553 553
554 554 In [1]: %reset -f
555 555
556 556 In [2]: alpha = 123
557 557
558 558 In [3]: beta = 'beta'
559 559
560 560 In [4]: %who int
561 561 alpha
562 562
563 563 In [5]: %who str
564 564 beta
565 565
566 566 In [6]: %whos
567 567 Variable Type Data/Info
568 568 ----------------------------
569 569 alpha int 123
570 570 beta str beta
571 571
572 572 In [7]: %who_ls
573 573 Out[7]: ['alpha', 'beta']
574 574 """
575 575
576 576 def test_whos():
577 577 """Check that whos is protected against objects where repr() fails."""
578 578 class A(object):
579 579 def __repr__(self):
580 580 raise Exception()
581 581 _ip.user_ns['a'] = A()
582 582 _ip.magic("whos")
583 583
584 584 def doctest_precision():
585 585 """doctest for %precision
586 586
587 587 In [1]: f = get_ipython().display_formatter.formatters['text/plain']
588 588
589 589 In [2]: %precision 5
590 590 Out[2]: '%.5f'
591 591
592 592 In [3]: f.float_format
593 593 Out[3]: '%.5f'
594 594
595 595 In [4]: %precision %e
596 596 Out[4]: '%e'
597 597
598 598 In [5]: f(3.1415927)
599 599 Out[5]: '3.141593e+00'
600 600 """
601 601
602 602 def test_psearch():
603 603 with tt.AssertPrints("dict.fromkeys"):
604 604 _ip.run_cell("dict.fr*?")
605 with tt.AssertPrints("π.is_integer"):
606 _ip.run_cell("π = 3.14;\nπ.is_integ*?")
605 607
606 608 def test_timeit_shlex():
607 609 """test shlex issues with timeit (#1109)"""
608 610 _ip.ex("def f(*a,**kw): pass")
609 611 _ip.magic('timeit -n1 "this is a bug".count(" ")')
610 612 _ip.magic('timeit -r1 -n1 f(" ", 1)')
611 613 _ip.magic('timeit -r1 -n1 f(" ", 1, " ", 2, " ")')
612 614 _ip.magic('timeit -r1 -n1 ("a " + "b")')
613 615 _ip.magic('timeit -r1 -n1 f("a " + "b")')
614 616 _ip.magic('timeit -r1 -n1 f("a " + "b ")')
615 617
616 618
617 619 def test_timeit_special_syntax():
618 620 "Test %%timeit with IPython special syntax"
619 621 @register_line_magic
620 622 def lmagic(line):
621 623 ip = get_ipython()
622 624 ip.user_ns['lmagic_out'] = line
623 625
624 626 # line mode test
625 627 _ip.run_line_magic('timeit', '-n1 -r1 %lmagic my line')
626 628 nt.assert_equal(_ip.user_ns['lmagic_out'], 'my line')
627 629 # cell mode test
628 630 _ip.run_cell_magic('timeit', '-n1 -r1', '%lmagic my line2')
629 631 nt.assert_equal(_ip.user_ns['lmagic_out'], 'my line2')
630 632
631 633 def test_timeit_return():
632 634 """
633 635 test whether timeit -o return object
634 636 """
635 637
636 638 res = _ip.run_line_magic('timeit','-n10 -r10 -o 1')
637 639 assert(res is not None)
638 640
639 641 def test_timeit_quiet():
640 642 """
641 643 test quiet option of timeit magic
642 644 """
643 645 with tt.AssertNotPrints("loops"):
644 646 _ip.run_cell("%timeit -n1 -r1 -q 1")
645 647
646 648 def test_timeit_return_quiet():
647 649 with tt.AssertNotPrints("loops"):
648 650 res = _ip.run_line_magic('timeit', '-n1 -r1 -q -o 1')
649 651 assert (res is not None)
650 652
651 653 def test_timeit_invalid_return():
652 654 with nt.assert_raises_regex(SyntaxError, "outside function"):
653 655 _ip.run_line_magic('timeit', 'return')
654 656
655 657 @dec.skipif(execution.profile is None)
656 658 def test_prun_special_syntax():
657 659 "Test %%prun with IPython special syntax"
658 660 @register_line_magic
659 661 def lmagic(line):
660 662 ip = get_ipython()
661 663 ip.user_ns['lmagic_out'] = line
662 664
663 665 # line mode test
664 666 _ip.run_line_magic('prun', '-q %lmagic my line')
665 667 nt.assert_equal(_ip.user_ns['lmagic_out'], 'my line')
666 668 # cell mode test
667 669 _ip.run_cell_magic('prun', '-q', '%lmagic my line2')
668 670 nt.assert_equal(_ip.user_ns['lmagic_out'], 'my line2')
669 671
670 672 @dec.skipif(execution.profile is None)
671 673 def test_prun_quotes():
672 674 "Test that prun does not clobber string escapes (GH #1302)"
673 675 _ip.magic(r"prun -q x = '\t'")
674 676 nt.assert_equal(_ip.user_ns['x'], '\t')
675 677
676 678 def test_extension():
677 679 # Debugging information for failures of this test
678 680 print('sys.path:')
679 681 for p in sys.path:
680 682 print(' ', p)
681 683 print('CWD', os.getcwd())
682 684
683 685 nt.assert_raises(ImportError, _ip.magic, "load_ext daft_extension")
684 686 daft_path = os.path.join(os.path.dirname(__file__), "daft_extension")
685 687 sys.path.insert(0, daft_path)
686 688 try:
687 689 _ip.user_ns.pop('arq', None)
688 690 invalidate_caches() # Clear import caches
689 691 _ip.magic("load_ext daft_extension")
690 692 nt.assert_equal(_ip.user_ns['arq'], 185)
691 693 _ip.magic("unload_ext daft_extension")
692 694 assert 'arq' not in _ip.user_ns
693 695 finally:
694 696 sys.path.remove(daft_path)
695 697
696 698
697 699 def test_notebook_export_json():
698 700 _ip = get_ipython()
699 701 _ip.history_manager.reset() # Clear any existing history.
700 702 cmds = [u"a=1", u"def b():\n return a**2", u"print('noël, été', b())"]
701 703 for i, cmd in enumerate(cmds, start=1):
702 704 _ip.history_manager.store_inputs(i, cmd)
703 705 with TemporaryDirectory() as td:
704 706 outfile = os.path.join(td, "nb.ipynb")
705 707 _ip.magic("notebook -e %s" % outfile)
706 708
707 709
708 710 class TestEnv(TestCase):
709 711
710 712 def test_env(self):
711 713 env = _ip.magic("env")
712 714 self.assertTrue(isinstance(env, dict))
713 715
714 716 def test_env_secret(self):
715 717 env = _ip.magic("env")
716 718 hidden = "<hidden>"
717 719 with mock.patch.dict(
718 720 os.environ,
719 721 {
720 722 "API_KEY": "abc123",
721 723 "SECRET_THING": "ssshhh",
722 724 "JUPYTER_TOKEN": "",
723 725 "VAR": "abc"
724 726 }
725 727 ):
726 728 env = _ip.magic("env")
727 729 assert env["API_KEY"] == hidden
728 730 assert env["SECRET_THING"] == hidden
729 731 assert env["JUPYTER_TOKEN"] == hidden
730 732 assert env["VAR"] == "abc"
731 733
732 734 def test_env_get_set_simple(self):
733 735 env = _ip.magic("env var val1")
734 736 self.assertEqual(env, None)
735 737 self.assertEqual(os.environ['var'], 'val1')
736 738 self.assertEqual(_ip.magic("env var"), 'val1')
737 739 env = _ip.magic("env var=val2")
738 740 self.assertEqual(env, None)
739 741 self.assertEqual(os.environ['var'], 'val2')
740 742
741 743 def test_env_get_set_complex(self):
742 744 env = _ip.magic("env var 'val1 '' 'val2")
743 745 self.assertEqual(env, None)
744 746 self.assertEqual(os.environ['var'], "'val1 '' 'val2")
745 747 self.assertEqual(_ip.magic("env var"), "'val1 '' 'val2")
746 748 env = _ip.magic('env var=val2 val3="val4')
747 749 self.assertEqual(env, None)
748 750 self.assertEqual(os.environ['var'], 'val2 val3="val4')
749 751
750 752 def test_env_set_bad_input(self):
751 753 self.assertRaises(UsageError, lambda: _ip.magic("set_env var"))
752 754
753 755 def test_env_set_whitespace(self):
754 756 self.assertRaises(UsageError, lambda: _ip.magic("env var A=B"))
755 757
756 758
757 759 class CellMagicTestCase(TestCase):
758 760
759 761 def check_ident(self, magic):
760 762 # Manually called, we get the result
761 763 out = _ip.run_cell_magic(magic, 'a', 'b')
762 764 nt.assert_equal(out, ('a','b'))
763 765 # Via run_cell, it goes into the user's namespace via displayhook
764 766 _ip.run_cell('%%' + magic +' c\nd\n')
765 767 nt.assert_equal(_ip.user_ns['_'], ('c','d\n'))
766 768
767 769 def test_cell_magic_func_deco(self):
768 770 "Cell magic using simple decorator"
769 771 @register_cell_magic
770 772 def cellm(line, cell):
771 773 return line, cell
772 774
773 775 self.check_ident('cellm')
774 776
775 777 def test_cell_magic_reg(self):
776 778 "Cell magic manually registered"
777 779 def cellm(line, cell):
778 780 return line, cell
779 781
780 782 _ip.register_magic_function(cellm, 'cell', 'cellm2')
781 783 self.check_ident('cellm2')
782 784
783 785 def test_cell_magic_class(self):
784 786 "Cell magics declared via a class"
785 787 @magics_class
786 788 class MyMagics(Magics):
787 789
788 790 @cell_magic
789 791 def cellm3(self, line, cell):
790 792 return line, cell
791 793
792 794 _ip.register_magics(MyMagics)
793 795 self.check_ident('cellm3')
794 796
795 797 def test_cell_magic_class2(self):
796 798 "Cell magics declared via a class, #2"
797 799 @magics_class
798 800 class MyMagics2(Magics):
799 801
800 802 @cell_magic('cellm4')
801 803 def cellm33(self, line, cell):
802 804 return line, cell
803 805
804 806 _ip.register_magics(MyMagics2)
805 807 self.check_ident('cellm4')
806 808 # Check that nothing is registered as 'cellm33'
807 809 c33 = _ip.find_cell_magic('cellm33')
808 810 nt.assert_equal(c33, None)
809 811
810 812 def test_file():
811 813 """Basic %%writefile"""
812 814 ip = get_ipython()
813 815 with TemporaryDirectory() as td:
814 816 fname = os.path.join(td, 'file1')
815 817 ip.run_cell_magic("writefile", fname, u'\n'.join([
816 818 'line1',
817 819 'line2',
818 820 ]))
819 821 with open(fname) as f:
820 822 s = f.read()
821 823 nt.assert_in('line1\n', s)
822 824 nt.assert_in('line2', s)
823 825
824 826 @dec.skip_win32
825 827 def test_file_single_quote():
826 828 """Basic %%writefile with embedded single quotes"""
827 829 ip = get_ipython()
828 830 with TemporaryDirectory() as td:
829 831 fname = os.path.join(td, '\'file1\'')
830 832 ip.run_cell_magic("writefile", fname, u'\n'.join([
831 833 'line1',
832 834 'line2',
833 835 ]))
834 836 with open(fname) as f:
835 837 s = f.read()
836 838 nt.assert_in('line1\n', s)
837 839 nt.assert_in('line2', s)
838 840
839 841 @dec.skip_win32
840 842 def test_file_double_quote():
841 843 """Basic %%writefile with embedded double quotes"""
842 844 ip = get_ipython()
843 845 with TemporaryDirectory() as td:
844 846 fname = os.path.join(td, '"file1"')
845 847 ip.run_cell_magic("writefile", fname, u'\n'.join([
846 848 'line1',
847 849 'line2',
848 850 ]))
849 851 with open(fname) as f:
850 852 s = f.read()
851 853 nt.assert_in('line1\n', s)
852 854 nt.assert_in('line2', s)
853 855
854 856 def test_file_var_expand():
855 857 """%%writefile $filename"""
856 858 ip = get_ipython()
857 859 with TemporaryDirectory() as td:
858 860 fname = os.path.join(td, 'file1')
859 861 ip.user_ns['filename'] = fname
860 862 ip.run_cell_magic("writefile", '$filename', u'\n'.join([
861 863 'line1',
862 864 'line2',
863 865 ]))
864 866 with open(fname) as f:
865 867 s = f.read()
866 868 nt.assert_in('line1\n', s)
867 869 nt.assert_in('line2', s)
868 870
869 871 def test_file_unicode():
870 872 """%%writefile with unicode cell"""
871 873 ip = get_ipython()
872 874 with TemporaryDirectory() as td:
873 875 fname = os.path.join(td, 'file1')
874 876 ip.run_cell_magic("writefile", fname, u'\n'.join([
875 877 u'liné1',
876 878 u'liné2',
877 879 ]))
878 880 with io.open(fname, encoding='utf-8') as f:
879 881 s = f.read()
880 882 nt.assert_in(u'liné1\n', s)
881 883 nt.assert_in(u'liné2', s)
882 884
883 885 def test_file_amend():
884 886 """%%writefile -a amends files"""
885 887 ip = get_ipython()
886 888 with TemporaryDirectory() as td:
887 889 fname = os.path.join(td, 'file2')
888 890 ip.run_cell_magic("writefile", fname, u'\n'.join([
889 891 'line1',
890 892 'line2',
891 893 ]))
892 894 ip.run_cell_magic("writefile", "-a %s" % fname, u'\n'.join([
893 895 'line3',
894 896 'line4',
895 897 ]))
896 898 with open(fname) as f:
897 899 s = f.read()
898 900 nt.assert_in('line1\n', s)
899 901 nt.assert_in('line3\n', s)
900 902
901 903 def test_file_spaces():
902 904 """%%file with spaces in filename"""
903 905 ip = get_ipython()
904 906 with TemporaryWorkingDirectory() as td:
905 907 fname = "file name"
906 908 ip.run_cell_magic("file", '"%s"'%fname, u'\n'.join([
907 909 'line1',
908 910 'line2',
909 911 ]))
910 912 with open(fname) as f:
911 913 s = f.read()
912 914 nt.assert_in('line1\n', s)
913 915 nt.assert_in('line2', s)
914 916
915 917 def test_script_config():
916 918 ip = get_ipython()
917 919 ip.config.ScriptMagics.script_magics = ['whoda']
918 920 sm = script.ScriptMagics(shell=ip)
919 921 nt.assert_in('whoda', sm.magics['cell'])
920 922
921 923 @dec.skip_win32
922 924 def test_script_out():
923 925 ip = get_ipython()
924 926 ip.run_cell_magic("script", "--out output sh", "echo 'hi'")
925 927 nt.assert_equal(ip.user_ns['output'], 'hi\n')
926 928
927 929 @dec.skip_win32
928 930 def test_script_err():
929 931 ip = get_ipython()
930 932 ip.run_cell_magic("script", "--err error sh", "echo 'hello' >&2")
931 933 nt.assert_equal(ip.user_ns['error'], 'hello\n')
932 934
933 935 @dec.skip_win32
934 936 def test_script_out_err():
935 937 ip = get_ipython()
936 938 ip.run_cell_magic("script", "--out output --err error sh", "echo 'hi'\necho 'hello' >&2")
937 939 nt.assert_equal(ip.user_ns['output'], 'hi\n')
938 940 nt.assert_equal(ip.user_ns['error'], 'hello\n')
939 941
940 942 @dec.skip_win32
941 943 def test_script_bg_out():
942 944 ip = get_ipython()
943 945 ip.run_cell_magic("script", "--bg --out output sh", "echo 'hi'")
944 946
945 947 nt.assert_equal(ip.user_ns['output'].read(), b'hi\n')
946 948 ip.user_ns['output'].close()
947 949
948 950 @dec.skip_win32
949 951 def test_script_bg_err():
950 952 ip = get_ipython()
951 953 ip.run_cell_magic("script", "--bg --err error sh", "echo 'hello' >&2")
952 954 nt.assert_equal(ip.user_ns['error'].read(), b'hello\n')
953 955 ip.user_ns['error'].close()
954 956
955 957 @dec.skip_win32
956 958 def test_script_bg_out_err():
957 959 ip = get_ipython()
958 960 ip.run_cell_magic("script", "--bg --out output --err error sh", "echo 'hi'\necho 'hello' >&2")
959 961 nt.assert_equal(ip.user_ns['output'].read(), b'hi\n')
960 962 nt.assert_equal(ip.user_ns['error'].read(), b'hello\n')
961 963 ip.user_ns['output'].close()
962 964 ip.user_ns['error'].close()
963 965
964 966 def test_script_defaults():
965 967 ip = get_ipython()
966 968 for cmd in ['sh', 'bash', 'perl', 'ruby']:
967 969 try:
968 970 find_cmd(cmd)
969 971 except Exception:
970 972 pass
971 973 else:
972 974 nt.assert_in(cmd, ip.magics_manager.magics['cell'])
973 975
974 976
975 977 @magics_class
976 978 class FooFoo(Magics):
977 979 """class with both %foo and %%foo magics"""
978 980 @line_magic('foo')
979 981 def line_foo(self, line):
980 982 "I am line foo"
981 983 pass
982 984
983 985 @cell_magic("foo")
984 986 def cell_foo(self, line, cell):
985 987 "I am cell foo, not line foo"
986 988 pass
987 989
988 990 def test_line_cell_info():
989 991 """%%foo and %foo magics are distinguishable to inspect"""
990 992 ip = get_ipython()
991 993 ip.magics_manager.register(FooFoo)
992 994 oinfo = ip.object_inspect('foo')
993 995 nt.assert_true(oinfo['found'])
994 996 nt.assert_true(oinfo['ismagic'])
995 997
996 998 oinfo = ip.object_inspect('%%foo')
997 999 nt.assert_true(oinfo['found'])
998 1000 nt.assert_true(oinfo['ismagic'])
999 1001 nt.assert_equal(oinfo['docstring'], FooFoo.cell_foo.__doc__)
1000 1002
1001 1003 oinfo = ip.object_inspect('%foo')
1002 1004 nt.assert_true(oinfo['found'])
1003 1005 nt.assert_true(oinfo['ismagic'])
1004 1006 nt.assert_equal(oinfo['docstring'], FooFoo.line_foo.__doc__)
1005 1007
1006 1008 def test_multiple_magics():
1007 1009 ip = get_ipython()
1008 1010 foo1 = FooFoo(ip)
1009 1011 foo2 = FooFoo(ip)
1010 1012 mm = ip.magics_manager
1011 1013 mm.register(foo1)
1012 1014 nt.assert_true(mm.magics['line']['foo'].__self__ is foo1)
1013 1015 mm.register(foo2)
1014 1016 nt.assert_true(mm.magics['line']['foo'].__self__ is foo2)
1015 1017
1016 1018 def test_alias_magic():
1017 1019 """Test %alias_magic."""
1018 1020 ip = get_ipython()
1019 1021 mm = ip.magics_manager
1020 1022
1021 1023 # Basic operation: both cell and line magics are created, if possible.
1022 1024 ip.run_line_magic('alias_magic', 'timeit_alias timeit')
1023 1025 nt.assert_in('timeit_alias', mm.magics['line'])
1024 1026 nt.assert_in('timeit_alias', mm.magics['cell'])
1025 1027
1026 1028 # --cell is specified, line magic not created.
1027 1029 ip.run_line_magic('alias_magic', '--cell timeit_cell_alias timeit')
1028 1030 nt.assert_not_in('timeit_cell_alias', mm.magics['line'])
1029 1031 nt.assert_in('timeit_cell_alias', mm.magics['cell'])
1030 1032
1031 1033 # Test that line alias is created successfully.
1032 1034 ip.run_line_magic('alias_magic', '--line env_alias env')
1033 1035 nt.assert_equal(ip.run_line_magic('env', ''),
1034 1036 ip.run_line_magic('env_alias', ''))
1035 1037
1036 1038 # Test that line alias with parameters passed in is created successfully.
1037 1039 ip.run_line_magic('alias_magic', '--line history_alias history --params ' + shlex.quote('3'))
1038 1040 nt.assert_in('history_alias', mm.magics['line'])
1039 1041
1040 1042
1041 1043 def test_save():
1042 1044 """Test %save."""
1043 1045 ip = get_ipython()
1044 1046 ip.history_manager.reset() # Clear any existing history.
1045 1047 cmds = [u"a=1", u"def b():\n return a**2", u"print(a, b())"]
1046 1048 for i, cmd in enumerate(cmds, start=1):
1047 1049 ip.history_manager.store_inputs(i, cmd)
1048 1050 with TemporaryDirectory() as tmpdir:
1049 1051 file = os.path.join(tmpdir, "testsave.py")
1050 1052 ip.run_line_magic("save", "%s 1-10" % file)
1051 1053 with open(file) as f:
1052 1054 content = f.read()
1053 1055 nt.assert_equal(content.count(cmds[0]), 1)
1054 1056 nt.assert_in('coding: utf-8', content)
1055 1057 ip.run_line_magic("save", "-a %s 1-10" % file)
1056 1058 with open(file) as f:
1057 1059 content = f.read()
1058 1060 nt.assert_equal(content.count(cmds[0]), 2)
1059 1061 nt.assert_in('coding: utf-8', content)
1060 1062
1061 1063
1062 1064 def test_store():
1063 1065 """Test %store."""
1064 1066 ip = get_ipython()
1065 1067 ip.run_line_magic('load_ext', 'storemagic')
1066 1068
1067 1069 # make sure the storage is empty
1068 1070 ip.run_line_magic('store', '-z')
1069 1071 ip.user_ns['var'] = 42
1070 1072 ip.run_line_magic('store', 'var')
1071 1073 ip.user_ns['var'] = 39
1072 1074 ip.run_line_magic('store', '-r')
1073 1075 nt.assert_equal(ip.user_ns['var'], 42)
1074 1076
1075 1077 ip.run_line_magic('store', '-d var')
1076 1078 ip.user_ns['var'] = 39
1077 1079 ip.run_line_magic('store' , '-r')
1078 1080 nt.assert_equal(ip.user_ns['var'], 39)
1079 1081
1080 1082
1081 1083 def _run_edit_test(arg_s, exp_filename=None,
1082 1084 exp_lineno=-1,
1083 1085 exp_contents=None,
1084 1086 exp_is_temp=None):
1085 1087 ip = get_ipython()
1086 1088 M = code.CodeMagics(ip)
1087 1089 last_call = ['','']
1088 1090 opts,args = M.parse_options(arg_s,'prxn:')
1089 1091 filename, lineno, is_temp = M._find_edit_target(ip, args, opts, last_call)
1090 1092
1091 1093 if exp_filename is not None:
1092 1094 nt.assert_equal(exp_filename, filename)
1093 1095 if exp_contents is not None:
1094 1096 with io.open(filename, 'r', encoding='utf-8') as f:
1095 1097 contents = f.read()
1096 1098 nt.assert_equal(exp_contents, contents)
1097 1099 if exp_lineno != -1:
1098 1100 nt.assert_equal(exp_lineno, lineno)
1099 1101 if exp_is_temp is not None:
1100 1102 nt.assert_equal(exp_is_temp, is_temp)
1101 1103
1102 1104
1103 1105 def test_edit_interactive():
1104 1106 """%edit on interactively defined objects"""
1105 1107 ip = get_ipython()
1106 1108 n = ip.execution_count
1107 1109 ip.run_cell(u"def foo(): return 1", store_history=True)
1108 1110
1109 1111 try:
1110 1112 _run_edit_test("foo")
1111 1113 except code.InteractivelyDefined as e:
1112 1114 nt.assert_equal(e.index, n)
1113 1115 else:
1114 1116 raise AssertionError("Should have raised InteractivelyDefined")
1115 1117
1116 1118
1117 1119 def test_edit_cell():
1118 1120 """%edit [cell id]"""
1119 1121 ip = get_ipython()
1120 1122
1121 1123 ip.run_cell(u"def foo(): return 1", store_history=True)
1122 1124
1123 1125 # test
1124 1126 _run_edit_test("1", exp_contents=ip.user_ns['In'][1], exp_is_temp=True)
1125 1127
1126 1128 def test_bookmark():
1127 1129 ip = get_ipython()
1128 1130 ip.run_line_magic('bookmark', 'bmname')
1129 1131 with tt.AssertPrints('bmname'):
1130 1132 ip.run_line_magic('bookmark', '-l')
1131 1133 ip.run_line_magic('bookmark', '-d bmname')
1132 1134
1133 1135 def test_ls_magic():
1134 1136 ip = get_ipython()
1135 1137 json_formatter = ip.display_formatter.formatters['application/json']
1136 1138 json_formatter.enabled = True
1137 1139 lsmagic = ip.magic('lsmagic')
1138 1140 with warnings.catch_warnings(record=True) as w:
1139 1141 j = json_formatter(lsmagic)
1140 1142 nt.assert_equal(sorted(j), ['cell', 'line'])
1141 1143 nt.assert_equal(w, []) # no warnings
1142 1144
1143 1145 def test_strip_initial_indent():
1144 1146 def sii(s):
1145 1147 lines = s.splitlines()
1146 1148 return '\n'.join(code.strip_initial_indent(lines))
1147 1149
1148 1150 nt.assert_equal(sii(" a = 1\nb = 2"), "a = 1\nb = 2")
1149 1151 nt.assert_equal(sii(" a\n b\nc"), "a\n b\nc")
1150 1152 nt.assert_equal(sii("a\n b"), "a\n b")
1151 1153
1152 1154 def test_logging_magic_quiet_from_arg():
1153 1155 _ip.config.LoggingMagics.quiet = False
1154 1156 lm = logging.LoggingMagics(shell=_ip)
1155 1157 with TemporaryDirectory() as td:
1156 1158 try:
1157 1159 with tt.AssertNotPrints(re.compile("Activating.*")):
1158 1160 lm.logstart('-q {}'.format(
1159 1161 os.path.join(td, "quiet_from_arg.log")))
1160 1162 finally:
1161 1163 _ip.logger.logstop()
1162 1164
1163 1165 def test_logging_magic_quiet_from_config():
1164 1166 _ip.config.LoggingMagics.quiet = True
1165 1167 lm = logging.LoggingMagics(shell=_ip)
1166 1168 with TemporaryDirectory() as td:
1167 1169 try:
1168 1170 with tt.AssertNotPrints(re.compile("Activating.*")):
1169 1171 lm.logstart(os.path.join(td, "quiet_from_config.log"))
1170 1172 finally:
1171 1173 _ip.logger.logstop()
1172 1174
1173 1175
1174 1176 def test_logging_magic_not_quiet():
1175 1177 _ip.config.LoggingMagics.quiet = False
1176 1178 lm = logging.LoggingMagics(shell=_ip)
1177 1179 with TemporaryDirectory() as td:
1178 1180 try:
1179 1181 with tt.AssertPrints(re.compile("Activating.*")):
1180 1182 lm.logstart(os.path.join(td, "not_quiet.log"))
1181 1183 finally:
1182 1184 _ip.logger.logstop()
1183 1185
1184 1186
1185 1187 def test_time_no_var_expand():
1186 1188 _ip.user_ns['a'] = 5
1187 1189 _ip.user_ns['b'] = []
1188 1190 _ip.magic('time b.append("{a}")')
1189 1191 assert _ip.user_ns['b'] == ['{a}']
1190 1192
1191 1193
1192 1194 # this is slow, put at the end for local testing.
1193 1195 def test_timeit_arguments():
1194 1196 "Test valid timeit arguments, should not cause SyntaxError (GH #1269)"
1195 1197 if sys.version_info < (3,7):
1196 1198 _ip.magic("timeit -n1 -r1 ('#')")
1197 1199 else:
1198 1200 # 3.7 optimize no-op statement like above out, and complain there is
1199 1201 # nothing in the for loop.
1200 1202 _ip.magic("timeit -n1 -r1 a=('#')")
1201 1203
1202 1204
1203 1205 TEST_MODULE = """
1204 1206 print('Loaded my_tmp')
1205 1207 if __name__ == "__main__":
1206 1208 print('I just ran a script')
1207 1209 """
1208 1210
1209 1211
1210 1212 def test_run_module_from_import_hook():
1211 1213 "Test that a module can be loaded via an import hook"
1212 1214 with TemporaryDirectory() as tmpdir:
1213 1215 fullpath = os.path.join(tmpdir, 'my_tmp.py')
1214 1216 with open(fullpath, 'w') as f:
1215 1217 f.write(TEST_MODULE)
1216 1218
1217 1219 class MyTempImporter(object):
1218 1220 def __init__(self):
1219 1221 pass
1220 1222
1221 1223 def find_module(self, fullname, path=None):
1222 1224 if 'my_tmp' in fullname:
1223 1225 return self
1224 1226 return None
1225 1227
1226 1228 def load_module(self, name):
1227 1229 import imp
1228 1230 return imp.load_source('my_tmp', fullpath)
1229 1231
1230 1232 def get_code(self, fullname):
1231 1233 with open(fullpath, 'r') as f:
1232 1234 return compile(f.read(), 'foo', 'exec')
1233 1235
1234 1236 def is_package(self, __):
1235 1237 return False
1236 1238
1237 1239 sys.meta_path.insert(0, MyTempImporter())
1238 1240
1239 1241 with capture_output() as captured:
1240 1242 _ip.magic("run -m my_tmp")
1241 1243 _ip.run_cell("import my_tmp")
1242 1244
1243 1245 output = "Loaded my_tmp\nI just ran a script\nLoaded my_tmp\n"
1244 1246 nt.assert_equal(output, captured.stdout)
1245 1247
1246 1248 sys.meta_path.pop(0)
@@ -1,117 +1,127 b''
1 1 """Tests for input manipulation machinery."""
2 2
3 3 #-----------------------------------------------------------------------------
4 4 # Imports
5 5 #-----------------------------------------------------------------------------
6 6 import nose.tools as nt
7 7
8 8 from IPython.core.prefilter import AutocallChecker
9 9
10 10 #-----------------------------------------------------------------------------
11 11 # Tests
12 12 #-----------------------------------------------------------------------------
13 13
14 14 def test_prefilter():
15 15 """Test user input conversions"""
16 16
17 17 # pairs of (raw, expected correct) input
18 18 pairs = [ ('2+2','2+2'),
19 19 ]
20 20
21 21 for raw, correct in pairs:
22 22 nt.assert_equal(ip.prefilter(raw), correct)
23 23
24 24 def test_prefilter_shadowed():
25 25 def dummy_magic(line): pass
26 26
27 27 prev_automagic_state = ip.automagic
28 28 ip.automagic = True
29 29 ip.autocall = 0
30 30
31 31 try:
32 32 # These should not be transformed - they are shadowed by other names
33 33 for name in ['if', 'zip', 'get_ipython']: # keyword, builtin, global
34 34 ip.register_magic_function(dummy_magic, magic_name=name)
35 35 res = ip.prefilter(name+' foo')
36 36 nt.assert_equal(res, name+' foo')
37 37 del ip.magics_manager.magics['line'][name]
38 38
39 39 # These should be transformed
40 40 for name in ['fi', 'piz', 'nohtypi_teg']:
41 41 ip.register_magic_function(dummy_magic, magic_name=name)
42 42 res = ip.prefilter(name+' foo')
43 43 nt.assert_not_equal(res, name+' foo')
44 44 del ip.magics_manager.magics['line'][name]
45 45
46 46 finally:
47 47 ip.automagic = prev_automagic_state
48 48
49 49 def test_autocall_binops():
50 50 """See https://github.com/ipython/ipython/issues/81"""
51 51 ip.magic('autocall 2')
52 52 f = lambda x: x
53 53 ip.user_ns['f'] = f
54 54 try:
55 55 nt.assert_equal(ip.prefilter('f 1'),'f(1)')
56 56 for t in ['f +1', 'f -1']:
57 57 nt.assert_equal(ip.prefilter(t), t)
58 58
59 59 # Run tests again with a more permissive exclude_regexp, which will
60 60 # allow transformation of binary operations ('f -1' -> 'f(-1)').
61 61 pm = ip.prefilter_manager
62 62 ac = AutocallChecker(shell=pm.shell, prefilter_manager=pm,
63 63 config=pm.config)
64 64 try:
65 65 ac.priority = 1
66 66 ac.exclude_regexp = r'^[,&^\|\*/]|^is |^not |^in |^and |^or '
67 67 pm.sort_checkers()
68 68
69 69 nt.assert_equal(ip.prefilter('f -1'), 'f(-1)')
70 70 nt.assert_equal(ip.prefilter('f +1'), 'f(+1)')
71 71 finally:
72 72 pm.unregister_checker(ac)
73 73 finally:
74 74 ip.magic('autocall 0')
75 75 del ip.user_ns['f']
76 76
77 77
78 78 def test_issue_114():
79 79 """Check that multiline string literals don't expand as magic
80 80 see http://github.com/ipython/ipython/issues/114"""
81 81
82 82 template = '"""\n%s\n"""'
83 83 # Store the current value of multi_line_specials and turn it off before
84 84 # running test, since it could be true (case in which the test doesn't make
85 85 # sense, as multiline string literals *will* expand as magic in that case).
86 86 msp = ip.prefilter_manager.multi_line_specials
87 87 ip.prefilter_manager.multi_line_specials = False
88 88 try:
89 89 for mgk in ip.magics_manager.lsmagic()['line']:
90 90 raw = template % mgk
91 91 nt.assert_equal(ip.prefilter(raw), raw)
92 92 finally:
93 93 ip.prefilter_manager.multi_line_specials = msp
94 94
95 95
96 96 def test_prefilter_attribute_errors():
97 97 """Capture exceptions thrown by user objects on attribute access.
98 98
99 99 See http://github.com/ipython/ipython/issues/988."""
100 100
101 101 class X(object):
102 102 def __getattr__(self, k):
103 103 raise ValueError('broken object')
104 104 def __call__(self, x):
105 105 return x
106 106
107 107 # Create a callable broken object
108 108 ip.user_ns['x'] = X()
109 109 ip.magic('autocall 2')
110 110 try:
111 111 # Even if x throws an attribute error when looking at its rewrite
112 112 # attribute, we should not crash. So the test here is simply making
113 113 # the prefilter call and not having an exception.
114 114 ip.prefilter('x 1')
115 115 finally:
116 116 del ip.user_ns['x']
117 117 ip.magic('autocall 0')
118
119
120 def test_autocall_should_support_unicode():
121 ip.magic('autocall 2')
122 ip.user_ns['π'] = lambda x: x
123 try:
124 nt.assert_equal(ip.prefilter('π 3'),'π(3)')
125 finally:
126 ip.magic('autocall 0')
127 del ip.user_ns['π']
General Comments 0
You need to be logged in to leave comments. Login now