##// END OF EJS Templates
typo and reformat
Matthias Bussonnier -
Show More

The requested changes are too big and content was truncated. Show full diff

@@ -1,155 +1,156 b''
1 1 """
2 2 IPython: tools for interactive and parallel computing in Python.
3 3
4 4 https://ipython.org
5 5 """
6 6 #-----------------------------------------------------------------------------
7 7 # Copyright (c) 2008-2011, IPython Development Team.
8 8 # Copyright (c) 2001-2007, Fernando Perez <fernando.perez@colorado.edu>
9 9 # Copyright (c) 2001, Janko Hauser <jhauser@zscout.de>
10 10 # Copyright (c) 2001, Nathaniel Gray <n8gray@caltech.edu>
11 11 #
12 12 # Distributed under the terms of the Modified BSD License.
13 13 #
14 14 # The full license is in the file COPYING.txt, distributed with this software.
15 15 #-----------------------------------------------------------------------------
16 16
17 17 #-----------------------------------------------------------------------------
18 18 # Imports
19 19 #-----------------------------------------------------------------------------
20 20
21 21 import os
22 22 import sys
23 23
24 24 #-----------------------------------------------------------------------------
25 25 # Setup everything
26 26 #-----------------------------------------------------------------------------
27 27
28 28 # Don't forget to also update setup.py when this changes!
29 29 if sys.version_info < (3, 8):
30 30 raise ImportError(
31 """
31 """
32 32 IPython 8+ supports Python 3.8 and above, following NEP 29.
33 33 When using Python 2.7, please install IPython 5.x LTS Long Term Support version.
34 34 Python 3.3 and 3.4 were supported up to IPython 6.x.
35 35 Python 3.5 was supported with IPython 7.0 to 7.9.
36 36 Python 3.6 was supported with IPython up to 7.16.
37 37 Python 3.7 was still supported with the 7.x branch.
38 38
39 39 See IPython `README.rst` file for more information:
40 40
41 41 https://github.com/ipython/ipython/blob/master/README.rst
42 42
43 """)
43 """
44 )
44 45
45 46 #-----------------------------------------------------------------------------
46 47 # Setup the top level names
47 48 #-----------------------------------------------------------------------------
48 49
49 50 from .core.getipython import get_ipython
50 51 from .core import release
51 52 from .core.application import Application
52 53 from .terminal.embed import embed
53 54
54 55 from .core.interactiveshell import InteractiveShell
55 56 from .utils.sysinfo import sys_info
56 57 from .utils.frame import extract_module_locals
57 58
58 59 # Release data
59 60 __author__ = '%s <%s>' % (release.author, release.author_email)
60 61 __license__ = release.license
61 62 __version__ = release.version
62 63 version_info = release.version_info
63 64 # list of CVEs that should have been patched in this release.
64 65 # this is informational and should not be relied upon.
65 66 __patched_cves__ = {"CVE-2022-21699"}
66 67
67 68
68 69 def embed_kernel(module=None, local_ns=None, **kwargs):
69 70 """Embed and start an IPython kernel in a given scope.
70 71
71 72 If you don't want the kernel to initialize the namespace
72 73 from the scope of the surrounding function,
73 74 and/or you want to load full IPython configuration,
74 75 you probably want `IPython.start_kernel()` instead.
75 76
76 77 Parameters
77 78 ----------
78 79 module : types.ModuleType, optional
79 80 The module to load into IPython globals (default: caller)
80 81 local_ns : dict, optional
81 82 The namespace to load into IPython user namespace (default: caller)
82 83 **kwargs : various, optional
83 84 Further keyword args are relayed to the IPKernelApp constructor,
84 85 allowing configuration of the Kernel. Will only have an effect
85 86 on the first embed_kernel call for a given process.
86 87 """
87 88
88 89 (caller_module, caller_locals) = extract_module_locals(1)
89 90 if module is None:
90 91 module = caller_module
91 92 if local_ns is None:
92 93 local_ns = caller_locals
93 94
94 95 # Only import .zmq when we really need it
95 96 from ipykernel.embed import embed_kernel as real_embed_kernel
96 97 real_embed_kernel(module=module, local_ns=local_ns, **kwargs)
97 98
98 99 def start_ipython(argv=None, **kwargs):
99 100 """Launch a normal IPython instance (as opposed to embedded)
100 101
101 102 `IPython.embed()` puts a shell in a particular calling scope,
102 103 such as a function or method for debugging purposes,
103 104 which is often not desirable.
104 105
105 106 `start_ipython()` does full, regular IPython initialization,
106 107 including loading startup files, configuration, etc.
107 108 much of which is skipped by `embed()`.
108 109
109 110 This is a public API method, and will survive implementation changes.
110 111
111 112 Parameters
112 113 ----------
113 114 argv : list or None, optional
114 115 If unspecified or None, IPython will parse command-line options from sys.argv.
115 116 To prevent any command-line parsing, pass an empty list: `argv=[]`.
116 117 user_ns : dict, optional
117 118 specify this dictionary to initialize the IPython user namespace with particular values.
118 119 **kwargs : various, optional
119 120 Any other kwargs will be passed to the Application constructor,
120 121 such as `config`.
121 122 """
122 123 from IPython.terminal.ipapp import launch_new_instance
123 124 return launch_new_instance(argv=argv, **kwargs)
124 125
125 126 def start_kernel(argv=None, **kwargs):
126 127 """Launch a normal IPython kernel instance (as opposed to embedded)
127 128
128 129 `IPython.embed_kernel()` puts a shell in a particular calling scope,
129 130 such as a function or method for debugging purposes,
130 131 which is often not desirable.
131 132
132 133 `start_kernel()` does full, regular IPython initialization,
133 134 including loading startup files, configuration, etc.
134 135 much of which is skipped by `embed()`.
135 136
136 137 Parameters
137 138 ----------
138 139 argv : list or None, optional
139 140 If unspecified or None, IPython will parse command-line options from sys.argv.
140 141 To prevent any command-line parsing, pass an empty list: `argv=[]`.
141 142 user_ns : dict, optional
142 143 specify this dictionary to initialize the IPython user namespace with particular values.
143 144 **kwargs : various, optional
144 145 Any other kwargs will be passed to the Application constructor,
145 146 such as `config`.
146 147 """
147 148 import warnings
148 149
149 150 warnings.warn(
150 151 "start_kernel is deprecated since IPython 8.0, use from `ipykernel.kernelapp.launch_new_instance`",
151 152 DeprecationWarning,
152 153 stacklevel=2,
153 154 )
154 155 from ipykernel.kernelapp import launch_new_instance
155 156 return launch_new_instance(argv=argv, **kwargs)
@@ -1,70 +1,70 b''
1 1 # encoding: utf-8
2 2 """
3 3 Autocall capabilities for IPython.core.
4 4
5 5 Authors:
6 6
7 7 * Brian Granger
8 8 * Fernando Perez
9 9 * Thomas Kluyver
10 10
11 11 Notes
12 12 -----
13 13 """
14 14
15 15 #-----------------------------------------------------------------------------
16 16 # Copyright (C) 2008-2011 The IPython Development Team
17 17 #
18 18 # Distributed under the terms of the BSD License. The full license is in
19 19 # the file COPYING, distributed as part of this software.
20 20 #-----------------------------------------------------------------------------
21 21
22 22 #-----------------------------------------------------------------------------
23 23 # Imports
24 24 #-----------------------------------------------------------------------------
25 25
26 26
27 27 #-----------------------------------------------------------------------------
28 28 # Code
29 29 #-----------------------------------------------------------------------------
30 30
31 31 class IPyAutocall(object):
32 32 """ Instances of this class are always autocalled
33 33
34 34 This happens regardless of 'autocall' variable state. Use this to
35 35 develop macro-like mechanisms.
36 36 """
37 37 _ip = None
38 38 rewrite = True
39 39 def __init__(self, ip=None):
40 40 self._ip = ip
41 41
42 42 def set_ip(self, ip):
43 """ Will be used to set _ip point to current ipython instance b/f call
43 """Will be used to set _ip point to current ipython instance b/f call
44 44
45 45 Override this method if you don't want this to happen.
46 46
47 47 """
48 48 self._ip = ip
49 49
50 50
51 51 class ExitAutocall(IPyAutocall):
52 52 """An autocallable object which will be added to the user namespace so that
53 53 exit, exit(), quit or quit() are all valid ways to close the shell."""
54 54 rewrite = False
55 55
56 56 def __call__(self):
57 57 self._ip.ask_exit()
58 58
59 59 class ZMQExitAutocall(ExitAutocall):
60 60 """Exit IPython. Autocallable, so it needn't be explicitly called.
61 61
62 62 Parameters
63 63 ----------
64 64 keep_kernel : bool
65 65 If True, leave the kernel alive. Otherwise, tell the kernel to exit too
66 66 (default).
67 67 """
68 68 def __call__(self, keep_kernel=False):
69 69 self._ip.keepkernel_on_exit = keep_kernel
70 70 self._ip.ask_exit()
@@ -1,2272 +1,2272 b''
1 1 """Completion for IPython.
2 2
3 3 This module started as fork of the rlcompleter module in the Python standard
4 4 library. The original enhancements made to rlcompleter have been sent
5 5 upstream and were accepted as of Python 2.3,
6 6
7 7 This module now support a wide variety of completion mechanism both available
8 8 for normal classic Python code, as well as completer for IPython specific
9 9 Syntax like magics.
10 10
11 11 Latex and Unicode completion
12 12 ============================
13 13
14 14 IPython and compatible frontends not only can complete your code, but can help
15 15 you to input a wide range of characters. In particular we allow you to insert
16 16 a unicode character using the tab completion mechanism.
17 17
18 18 Forward latex/unicode completion
19 19 --------------------------------
20 20
21 21 Forward completion allows you to easily type a unicode character using its latex
22 22 name, or unicode long description. To do so type a backslash follow by the
23 23 relevant name and press tab:
24 24
25 25
26 26 Using latex completion:
27 27
28 28 .. code::
29 29
30 30 \\alpha<tab>
31 31 α
32 32
33 33 or using unicode completion:
34 34
35 35
36 36 .. code::
37 37
38 38 \\GREEK SMALL LETTER ALPHA<tab>
39 39 α
40 40
41 41
42 42 Only valid Python identifiers will complete. Combining characters (like arrow or
43 43 dots) are also available, unlike latex they need to be put after the their
44 44 counterpart that is to say, ``F\\\\vec<tab>`` is correct, not ``\\\\vec<tab>F``.
45 45
46 46 Some browsers are known to display combining characters incorrectly.
47 47
48 48 Backward latex completion
49 49 -------------------------
50 50
51 51 It is sometime challenging to know how to type a character, if you are using
52 52 IPython, or any compatible frontend you can prepend backslash to the character
53 53 and press ``<tab>`` to expand it to its latex form.
54 54
55 55 .. code::
56 56
57 57 \\α<tab>
58 58 \\alpha
59 59
60 60
61 61 Both forward and backward completions can be deactivated by setting the
62 62 ``Completer.backslash_combining_completions`` option to ``False``.
63 63
64 64
65 65 Experimental
66 66 ============
67 67
68 68 Starting with IPython 6.0, this module can make use of the Jedi library to
69 69 generate completions both using static analysis of the code, and dynamically
70 70 inspecting multiple namespaces. Jedi is an autocompletion and static analysis
71 71 for Python. The APIs attached to this new mechanism is unstable and will
72 72 raise unless use in an :any:`provisionalcompleter` context manager.
73 73
74 74 You will find that the following are experimental:
75 75
76 76 - :any:`provisionalcompleter`
77 77 - :any:`IPCompleter.completions`
78 78 - :any:`Completion`
79 79 - :any:`rectify_completions`
80 80
81 81 .. note::
82 82
83 83 better name for :any:`rectify_completions` ?
84 84
85 85 We welcome any feedback on these new API, and we also encourage you to try this
86 86 module in debug mode (start IPython with ``--Completer.debug=True``) in order
87 87 to have extra logging information if :any:`jedi` is crashing, or if current
88 88 IPython completer pending deprecations are returning results not yet handled
89 89 by :any:`jedi`
90 90
91 91 Using Jedi for tab completion allow snippets like the following to work without
92 92 having to execute any code:
93 93
94 94 >>> myvar = ['hello', 42]
95 95 ... myvar[1].bi<tab>
96 96
97 97 Tab completion will be able to infer that ``myvar[1]`` is a real number without
98 98 executing any code unlike the previously available ``IPCompleter.greedy``
99 99 option.
100 100
101 101 Be sure to update :any:`jedi` to the latest stable version or to try the
102 102 current development version to get better completions.
103 103 """
104 104
105 105
106 106 # Copyright (c) IPython Development Team.
107 107 # Distributed under the terms of the Modified BSD License.
108 108 #
109 109 # Some of this code originated from rlcompleter in the Python standard library
110 110 # Copyright (C) 2001 Python Software Foundation, www.python.org
111 111
112 112
113 113 import builtins as builtin_mod
114 114 import glob
115 115 import inspect
116 116 import itertools
117 117 import keyword
118 118 import os
119 119 import re
120 120 import string
121 121 import sys
122 122 import time
123 123 import unicodedata
124 124 import uuid
125 125 import warnings
126 126 from contextlib import contextmanager
127 127 from importlib import import_module
128 128 from types import SimpleNamespace
129 129 from typing import Iterable, Iterator, List, Tuple, Union, Any, Sequence, Dict, NamedTuple, Pattern, Optional
130 130
131 131 from IPython.core.error import TryNext
132 132 from IPython.core.inputtransformer2 import ESC_MAGIC
133 133 from IPython.core.latex_symbols import latex_symbols, reverse_latex_symbol
134 134 from IPython.core.oinspect import InspectColors
135 135 from IPython.testing.skipdoctest import skip_doctest
136 136 from IPython.utils import generics
137 137 from IPython.utils.dir2 import dir2, get_real_method
138 138 from IPython.utils.path import ensure_dir_exists
139 139 from IPython.utils.process import arg_split
140 140 from traitlets import Bool, Enum, Int, List as ListTrait, Unicode, default, observe
141 141 from traitlets.config.configurable import Configurable
142 142
143 143 import __main__
144 144
145 145 # skip module docstests
146 146 __skip_doctest__ = True
147 147
148 148 try:
149 149 import jedi
150 150 jedi.settings.case_insensitive_completion = False
151 151 import jedi.api.helpers
152 152 import jedi.api.classes
153 153 JEDI_INSTALLED = True
154 154 except ImportError:
155 155 JEDI_INSTALLED = False
156 156 #-----------------------------------------------------------------------------
157 157 # Globals
158 158 #-----------------------------------------------------------------------------
159 159
160 160 # ranges where we have most of the valid unicode names. We could be more finer
161 161 # grained but is it worth it for performance While unicode have character in the
162 162 # range 0, 0x110000, we seem to have name for about 10% of those. (131808 as I
163 163 # write this). With below range we cover them all, with a density of ~67%
164 164 # biggest next gap we consider only adds up about 1% density and there are 600
165 165 # gaps that would need hard coding.
166 166 _UNICODE_RANGES = [(32, 0x3134b), (0xe0001, 0xe01f0)]
167 167
168 168 # Public API
169 169 __all__ = ['Completer','IPCompleter']
170 170
171 171 if sys.platform == 'win32':
172 172 PROTECTABLES = ' '
173 173 else:
174 174 PROTECTABLES = ' ()[]{}?=\\|;:\'#*"^&'
175 175
176 176 # Protect against returning an enormous number of completions which the frontend
177 177 # may have trouble processing.
178 178 MATCHES_LIMIT = 500
179 179
180 180
181 181 class ProvisionalCompleterWarning(FutureWarning):
182 182 """
183 183 Exception raise by an experimental feature in this module.
184 184
185 185 Wrap code in :any:`provisionalcompleter` context manager if you
186 186 are certain you want to use an unstable feature.
187 187 """
188 188 pass
189 189
190 190 warnings.filterwarnings('error', category=ProvisionalCompleterWarning)
191 191
192 192
193 193 @skip_doctest
194 194 @contextmanager
195 195 def provisionalcompleter(action='ignore'):
196 196 """
197 197 This context manager has to be used in any place where unstable completer
198 198 behavior and API may be called.
199 199
200 200 >>> with provisionalcompleter():
201 201 ... completer.do_experimental_things() # works
202 202
203 203 >>> completer.do_experimental_things() # raises.
204 204
205 205 .. note::
206 206
207 207 Unstable
208 208
209 209 By using this context manager you agree that the API in use may change
210 210 without warning, and that you won't complain if they do so.
211 211
212 212 You also understand that, if the API is not to your liking, you should report
213 213 a bug to explain your use case upstream.
214 214
215 215 We'll be happy to get your feedback, feature requests, and improvements on
216 216 any of the unstable APIs!
217 217 """
218 218 with warnings.catch_warnings():
219 219 warnings.filterwarnings(action, category=ProvisionalCompleterWarning)
220 220 yield
221 221
222 222
223 223 def has_open_quotes(s):
224 224 """Return whether a string has open quotes.
225 225
226 226 This simply counts whether the number of quote characters of either type in
227 227 the string is odd.
228 228
229 229 Returns
230 230 -------
231 231 If there is an open quote, the quote character is returned. Else, return
232 232 False.
233 233 """
234 234 # We check " first, then ', so complex cases with nested quotes will get
235 235 # the " to take precedence.
236 236 if s.count('"') % 2:
237 237 return '"'
238 238 elif s.count("'") % 2:
239 239 return "'"
240 240 else:
241 241 return False
242 242
243 243
244 244 def protect_filename(s, protectables=PROTECTABLES):
245 245 """Escape a string to protect certain characters."""
246 246 if set(s) & set(protectables):
247 247 if sys.platform == "win32":
248 248 return '"' + s + '"'
249 249 else:
250 250 return "".join(("\\" + c if c in protectables else c) for c in s)
251 251 else:
252 252 return s
253 253
254 254
255 255 def expand_user(path:str) -> Tuple[str, bool, str]:
256 256 """Expand ``~``-style usernames in strings.
257 257
258 258 This is similar to :func:`os.path.expanduser`, but it computes and returns
259 259 extra information that will be useful if the input was being used in
260 260 computing completions, and you wish to return the completions with the
261 261 original '~' instead of its expanded value.
262 262
263 263 Parameters
264 264 ----------
265 265 path : str
266 266 String to be expanded. If no ~ is present, the output is the same as the
267 267 input.
268 268
269 269 Returns
270 270 -------
271 271 newpath : str
272 272 Result of ~ expansion in the input path.
273 273 tilde_expand : bool
274 274 Whether any expansion was performed or not.
275 275 tilde_val : str
276 276 The value that ~ was replaced with.
277 277 """
278 278 # Default values
279 279 tilde_expand = False
280 280 tilde_val = ''
281 281 newpath = path
282 282
283 283 if path.startswith('~'):
284 284 tilde_expand = True
285 285 rest = len(path)-1
286 286 newpath = os.path.expanduser(path)
287 287 if rest:
288 288 tilde_val = newpath[:-rest]
289 289 else:
290 290 tilde_val = newpath
291 291
292 292 return newpath, tilde_expand, tilde_val
293 293
294 294
295 295 def compress_user(path:str, tilde_expand:bool, tilde_val:str) -> str:
296 296 """Does the opposite of expand_user, with its outputs.
297 297 """
298 298 if tilde_expand:
299 299 return path.replace(tilde_val, '~')
300 300 else:
301 301 return path
302 302
303 303
304 304 def completions_sorting_key(word):
305 305 """key for sorting completions
306 306
307 307 This does several things:
308 308
309 309 - Demote any completions starting with underscores to the end
310 310 - Insert any %magic and %%cellmagic completions in the alphabetical order
311 311 by their name
312 312 """
313 313 prio1, prio2 = 0, 0
314 314
315 315 if word.startswith('__'):
316 316 prio1 = 2
317 317 elif word.startswith('_'):
318 318 prio1 = 1
319 319
320 320 if word.endswith('='):
321 321 prio1 = -1
322 322
323 323 if word.startswith('%%'):
324 324 # If there's another % in there, this is something else, so leave it alone
325 325 if not "%" in word[2:]:
326 326 word = word[2:]
327 327 prio2 = 2
328 328 elif word.startswith('%'):
329 329 if not "%" in word[1:]:
330 330 word = word[1:]
331 331 prio2 = 1
332 332
333 333 return prio1, word, prio2
334 334
335 335
336 336 class _FakeJediCompletion:
337 337 """
338 338 This is a workaround to communicate to the UI that Jedi has crashed and to
339 339 report a bug. Will be used only id :any:`IPCompleter.debug` is set to true.
340 340
341 341 Added in IPython 6.0 so should likely be removed for 7.0
342 342
343 343 """
344 344
345 345 def __init__(self, name):
346 346
347 347 self.name = name
348 348 self.complete = name
349 349 self.type = 'crashed'
350 350 self.name_with_symbols = name
351 351 self.signature = ''
352 352 self._origin = 'fake'
353 353
354 354 def __repr__(self):
355 355 return '<Fake completion object jedi has crashed>'
356 356
357 357
358 358 class Completion:
359 359 """
360 360 Completion object used and return by IPython completers.
361 361
362 362 .. warning::
363 363
364 364 Unstable
365 365
366 366 This function is unstable, API may change without warning.
367 367 It will also raise unless use in proper context manager.
368 368
369 369 This act as a middle ground :any:`Completion` object between the
370 370 :any:`jedi.api.classes.Completion` object and the Prompt Toolkit completion
371 371 object. While Jedi need a lot of information about evaluator and how the
372 372 code should be ran/inspected, PromptToolkit (and other frontend) mostly
373 373 need user facing information.
374 374
375 375 - Which range should be replaced replaced by what.
376 376 - Some metadata (like completion type), or meta information to displayed to
377 377 the use user.
378 378
379 379 For debugging purpose we can also store the origin of the completion (``jedi``,
380 380 ``IPython.python_matches``, ``IPython.magics_matches``...).
381 381 """
382 382
383 383 __slots__ = ['start', 'end', 'text', 'type', 'signature', '_origin']
384 384
385 385 def __init__(self, start: int, end: int, text: str, *, type: str=None, _origin='', signature='') -> None:
386 386 warnings.warn("``Completion`` is a provisional API (as of IPython 6.0). "
387 387 "It may change without warnings. "
388 388 "Use in corresponding context manager.",
389 389 category=ProvisionalCompleterWarning, stacklevel=2)
390 390
391 391 self.start = start
392 392 self.end = end
393 393 self.text = text
394 394 self.type = type
395 395 self.signature = signature
396 396 self._origin = _origin
397 397
398 398 def __repr__(self):
399 399 return '<Completion start=%s end=%s text=%r type=%r, signature=%r,>' % \
400 400 (self.start, self.end, self.text, self.type or '?', self.signature or '?')
401 401
402 402 def __eq__(self, other)->Bool:
403 403 """
404 404 Equality and hash do not hash the type (as some completer may not be
405 405 able to infer the type), but are use to (partially) de-duplicate
406 406 completion.
407 407
408 408 Completely de-duplicating completion is a bit tricker that just
409 409 comparing as it depends on surrounding text, which Completions are not
410 410 aware of.
411 411 """
412 412 return self.start == other.start and \
413 413 self.end == other.end and \
414 414 self.text == other.text
415 415
416 416 def __hash__(self):
417 417 return hash((self.start, self.end, self.text))
418 418
419 419
420 420 _IC = Iterable[Completion]
421 421
422 422
423 423 def _deduplicate_completions(text: str, completions: _IC)-> _IC:
424 424 """
425 425 Deduplicate a set of completions.
426 426
427 427 .. warning::
428 428
429 429 Unstable
430 430
431 431 This function is unstable, API may change without warning.
432 432
433 433 Parameters
434 434 ----------
435 435 text : str
436 436 text that should be completed.
437 437 completions : Iterator[Completion]
438 438 iterator over the completions to deduplicate
439 439
440 440 Yields
441 441 ------
442 442 `Completions` objects
443 443 Completions coming from multiple sources, may be different but end up having
444 444 the same effect when applied to ``text``. If this is the case, this will
445 445 consider completions as equal and only emit the first encountered.
446 446 Not folded in `completions()` yet for debugging purpose, and to detect when
447 447 the IPython completer does return things that Jedi does not, but should be
448 448 at some point.
449 449 """
450 450 completions = list(completions)
451 451 if not completions:
452 452 return
453 453
454 454 new_start = min(c.start for c in completions)
455 455 new_end = max(c.end for c in completions)
456 456
457 457 seen = set()
458 458 for c in completions:
459 459 new_text = text[new_start:c.start] + c.text + text[c.end:new_end]
460 460 if new_text not in seen:
461 461 yield c
462 462 seen.add(new_text)
463 463
464 464
465 465 def rectify_completions(text: str, completions: _IC, *, _debug: bool = False) -> _IC:
466 466 """
467 467 Rectify a set of completions to all have the same ``start`` and ``end``
468 468
469 469 .. warning::
470 470
471 471 Unstable
472 472
473 473 This function is unstable, API may change without warning.
474 474 It will also raise unless use in proper context manager.
475 475
476 476 Parameters
477 477 ----------
478 478 text : str
479 479 text that should be completed.
480 480 completions : Iterator[Completion]
481 481 iterator over the completions to rectify
482 482 _debug : bool
483 483 Log failed completion
484 484
485 485 Notes
486 486 -----
487 487 :any:`jedi.api.classes.Completion` s returned by Jedi may not have the same start and end, though
488 488 the Jupyter Protocol requires them to behave like so. This will readjust
489 489 the completion to have the same ``start`` and ``end`` by padding both
490 490 extremities with surrounding text.
491 491
492 492 During stabilisation should support a ``_debug`` option to log which
493 493 completion are return by the IPython completer and not found in Jedi in
494 494 order to make upstream bug report.
495 495 """
496 496 warnings.warn("`rectify_completions` is a provisional API (as of IPython 6.0). "
497 497 "It may change without warnings. "
498 498 "Use in corresponding context manager.",
499 499 category=ProvisionalCompleterWarning, stacklevel=2)
500 500
501 501 completions = list(completions)
502 502 if not completions:
503 503 return
504 504 starts = (c.start for c in completions)
505 505 ends = (c.end for c in completions)
506 506
507 507 new_start = min(starts)
508 508 new_end = max(ends)
509 509
510 510 seen_jedi = set()
511 511 seen_python_matches = set()
512 512 for c in completions:
513 513 new_text = text[new_start:c.start] + c.text + text[c.end:new_end]
514 514 if c._origin == 'jedi':
515 515 seen_jedi.add(new_text)
516 516 elif c._origin == 'IPCompleter.python_matches':
517 517 seen_python_matches.add(new_text)
518 518 yield Completion(new_start, new_end, new_text, type=c.type, _origin=c._origin, signature=c.signature)
519 519 diff = seen_python_matches.difference(seen_jedi)
520 520 if diff and _debug:
521 521 print('IPython.python matches have extras:', diff)
522 522
523 523
524 524 if sys.platform == 'win32':
525 525 DELIMS = ' \t\n`!@#$^&*()=+[{]}|;\'",<>?'
526 526 else:
527 527 DELIMS = ' \t\n`!@#$^&*()=+[{]}\\|;:\'",<>?'
528 528
529 529 GREEDY_DELIMS = ' =\r\n'
530 530
531 531
532 532 class CompletionSplitter(object):
533 533 """An object to split an input line in a manner similar to readline.
534 534
535 535 By having our own implementation, we can expose readline-like completion in
536 536 a uniform manner to all frontends. This object only needs to be given the
537 537 line of text to be split and the cursor position on said line, and it
538 538 returns the 'word' to be completed on at the cursor after splitting the
539 539 entire line.
540 540
541 541 What characters are used as splitting delimiters can be controlled by
542 542 setting the ``delims`` attribute (this is a property that internally
543 543 automatically builds the necessary regular expression)"""
544 544
545 545 # Private interface
546 546
547 547 # A string of delimiter characters. The default value makes sense for
548 548 # IPython's most typical usage patterns.
549 549 _delims = DELIMS
550 550
551 551 # The expression (a normal string) to be compiled into a regular expression
552 552 # for actual splitting. We store it as an attribute mostly for ease of
553 553 # debugging, since this type of code can be so tricky to debug.
554 554 _delim_expr = None
555 555
556 556 # The regular expression that does the actual splitting
557 557 _delim_re = None
558 558
559 559 def __init__(self, delims=None):
560 560 delims = CompletionSplitter._delims if delims is None else delims
561 561 self.delims = delims
562 562
563 563 @property
564 564 def delims(self):
565 565 """Return the string of delimiter characters."""
566 566 return self._delims
567 567
568 568 @delims.setter
569 569 def delims(self, delims):
570 570 """Set the delimiters for line splitting."""
571 571 expr = '[' + ''.join('\\'+ c for c in delims) + ']'
572 572 self._delim_re = re.compile(expr)
573 573 self._delims = delims
574 574 self._delim_expr = expr
575 575
576 576 def split_line(self, line, cursor_pos=None):
577 577 """Split a line of text with a cursor at the given position.
578 578 """
579 579 l = line if cursor_pos is None else line[:cursor_pos]
580 580 return self._delim_re.split(l)[-1]
581 581
582 582
583 583
584 584 class Completer(Configurable):
585 585
586 586 greedy = Bool(False,
587 587 help="""Activate greedy completion
588 588 PENDING DEPRECATION. this is now mostly taken care of with Jedi.
589 589
590 590 This will enable completion on elements of lists, results of function calls, etc.,
591 591 but can be unsafe because the code is actually evaluated on TAB.
592 """
592 """,
593 593 ).tag(config=True)
594 594
595 595 use_jedi = Bool(default_value=JEDI_INSTALLED,
596 596 help="Experimental: Use Jedi to generate autocompletions. "
597 597 "Default to True if jedi is installed.").tag(config=True)
598 598
599 599 jedi_compute_type_timeout = Int(default_value=400,
600 600 help="""Experimental: restrict time (in milliseconds) during which Jedi can compute types.
601 601 Set to 0 to stop computing types. Non-zero value lower than 100ms may hurt
602 602 performance by preventing jedi to build its cache.
603 603 """).tag(config=True)
604 604
605 605 debug = Bool(default_value=False,
606 606 help='Enable debug for the Completer. Mostly print extra '
607 607 'information for experimental jedi integration.')\
608 608 .tag(config=True)
609 609
610 610 backslash_combining_completions = Bool(True,
611 611 help="Enable unicode completions, e.g. \\alpha<tab> . "
612 612 "Includes completion of latex commands, unicode names, and expanding "
613 613 "unicode characters back to latex commands.").tag(config=True)
614 614
615 615 def __init__(self, namespace=None, global_namespace=None, **kwargs):
616 616 """Create a new completer for the command line.
617 617
618 618 Completer(namespace=ns, global_namespace=ns2) -> completer instance.
619 619
620 620 If unspecified, the default namespace where completions are performed
621 621 is __main__ (technically, __main__.__dict__). Namespaces should be
622 622 given as dictionaries.
623 623
624 624 An optional second namespace can be given. This allows the completer
625 625 to handle cases where both the local and global scopes need to be
626 626 distinguished.
627 627 """
628 628
629 629 # Don't bind to namespace quite yet, but flag whether the user wants a
630 630 # specific namespace or to use __main__.__dict__. This will allow us
631 631 # to bind to __main__.__dict__ at completion time, not now.
632 632 if namespace is None:
633 633 self.use_main_ns = True
634 634 else:
635 635 self.use_main_ns = False
636 636 self.namespace = namespace
637 637
638 638 # The global namespace, if given, can be bound directly
639 639 if global_namespace is None:
640 640 self.global_namespace = {}
641 641 else:
642 642 self.global_namespace = global_namespace
643 643
644 644 self.custom_matchers = []
645 645
646 646 super(Completer, self).__init__(**kwargs)
647 647
648 648 def complete(self, text, state):
649 649 """Return the next possible completion for 'text'.
650 650
651 651 This is called successively with state == 0, 1, 2, ... until it
652 652 returns None. The completion should begin with 'text'.
653 653
654 654 """
655 655 if self.use_main_ns:
656 656 self.namespace = __main__.__dict__
657 657
658 658 if state == 0:
659 659 if "." in text:
660 660 self.matches = self.attr_matches(text)
661 661 else:
662 662 self.matches = self.global_matches(text)
663 663 try:
664 664 return self.matches[state]
665 665 except IndexError:
666 666 return None
667 667
668 668 def global_matches(self, text):
669 669 """Compute matches when text is a simple name.
670 670
671 671 Return a list of all keywords, built-in functions and names currently
672 672 defined in self.namespace or self.global_namespace that match.
673 673
674 674 """
675 675 matches = []
676 676 match_append = matches.append
677 677 n = len(text)
678 678 for lst in [keyword.kwlist,
679 679 builtin_mod.__dict__.keys(),
680 680 self.namespace.keys(),
681 681 self.global_namespace.keys()]:
682 682 for word in lst:
683 683 if word[:n] == text and word != "__builtins__":
684 684 match_append(word)
685 685
686 686 snake_case_re = re.compile(r"[^_]+(_[^_]+)+?\Z")
687 687 for lst in [self.namespace.keys(),
688 688 self.global_namespace.keys()]:
689 689 shortened = {"_".join([sub[0] for sub in word.split('_')]) : word
690 690 for word in lst if snake_case_re.match(word)}
691 691 for word in shortened.keys():
692 692 if word[:n] == text and word != "__builtins__":
693 693 match_append(shortened[word])
694 694 return matches
695 695
696 696 def attr_matches(self, text):
697 697 """Compute matches when text contains a dot.
698 698
699 699 Assuming the text is of the form NAME.NAME....[NAME], and is
700 700 evaluatable in self.namespace or self.global_namespace, it will be
701 701 evaluated and its attributes (as revealed by dir()) are used as
702 702 possible completions. (For class instances, class members are
703 703 also considered.)
704 704
705 705 WARNING: this can still invoke arbitrary C code, if an object
706 706 with a __getattr__ hook is evaluated.
707 707
708 708 """
709 709
710 710 # Another option, seems to work great. Catches things like ''.<tab>
711 711 m = re.match(r"(\S+(\.\w+)*)\.(\w*)$", text)
712 712
713 713 if m:
714 714 expr, attr = m.group(1, 3)
715 715 elif self.greedy:
716 716 m2 = re.match(r"(.+)\.(\w*)$", self.line_buffer)
717 717 if not m2:
718 718 return []
719 719 expr, attr = m2.group(1,2)
720 720 else:
721 721 return []
722 722
723 723 try:
724 724 obj = eval(expr, self.namespace)
725 725 except:
726 726 try:
727 727 obj = eval(expr, self.global_namespace)
728 728 except:
729 729 return []
730 730
731 731 if self.limit_to__all__ and hasattr(obj, '__all__'):
732 732 words = get__all__entries(obj)
733 733 else:
734 734 words = dir2(obj)
735 735
736 736 try:
737 737 words = generics.complete_object(obj, words)
738 738 except TryNext:
739 739 pass
740 740 except AssertionError:
741 741 raise
742 742 except Exception:
743 743 # Silence errors from completion function
744 744 #raise # dbg
745 745 pass
746 746 # Build match list to return
747 747 n = len(attr)
748 748 return [u"%s.%s" % (expr, w) for w in words if w[:n] == attr ]
749 749
750 750
751 751 def get__all__entries(obj):
752 752 """returns the strings in the __all__ attribute"""
753 753 try:
754 754 words = getattr(obj, '__all__')
755 755 except:
756 756 return []
757 757
758 758 return [w for w in words if isinstance(w, str)]
759 759
760 760
761 761 def match_dict_keys(keys: List[Union[str, bytes, Tuple[Union[str, bytes]]]], prefix: str, delims: str,
762 762 extra_prefix: Optional[Tuple[str, bytes]]=None) -> Tuple[str, int, List[str]]:
763 763 """Used by dict_key_matches, matching the prefix to a list of keys
764 764
765 765 Parameters
766 766 ----------
767 767 keys
768 768 list of keys in dictionary currently being completed.
769 769 prefix
770 770 Part of the text already typed by the user. E.g. `mydict[b'fo`
771 771 delims
772 772 String of delimiters to consider when finding the current key.
773 773 extra_prefix : optional
774 774 Part of the text already typed in multi-key index cases. E.g. for
775 775 `mydict['foo', "bar", 'b`, this would be `('foo', 'bar')`.
776 776
777 777 Returns
778 778 -------
779 779 A tuple of three elements: ``quote``, ``token_start``, ``matched``, with
780 780 ``quote`` being the quote that need to be used to close current string.
781 781 ``token_start`` the position where the replacement should start occurring,
782 782 ``matches`` a list of replacement/completion
783 783
784 784 """
785 785 prefix_tuple = extra_prefix if extra_prefix else ()
786 786 Nprefix = len(prefix_tuple)
787 787 def filter_prefix_tuple(key):
788 788 # Reject too short keys
789 789 if len(key) <= Nprefix:
790 790 return False
791 791 # Reject keys with non str/bytes in it
792 792 for k in key:
793 793 if not isinstance(k, (str, bytes)):
794 794 return False
795 795 # Reject keys that do not match the prefix
796 796 for k, pt in zip(key, prefix_tuple):
797 797 if k != pt:
798 798 return False
799 799 # All checks passed!
800 800 return True
801 801
802 802 filtered_keys:List[Union[str,bytes]] = []
803 803 def _add_to_filtered_keys(key):
804 804 if isinstance(key, (str, bytes)):
805 805 filtered_keys.append(key)
806 806
807 807 for k in keys:
808 808 if isinstance(k, tuple):
809 809 if filter_prefix_tuple(k):
810 810 _add_to_filtered_keys(k[Nprefix])
811 811 else:
812 812 _add_to_filtered_keys(k)
813 813
814 814 if not prefix:
815 815 return '', 0, [repr(k) for k in filtered_keys]
816 816 quote_match = re.search('["\']', prefix)
817 817 assert quote_match is not None # silence mypy
818 818 quote = quote_match.group()
819 819 try:
820 820 prefix_str = eval(prefix + quote, {})
821 821 except Exception:
822 822 return '', 0, []
823 823
824 824 pattern = '[^' + ''.join('\\' + c for c in delims) + ']*$'
825 825 token_match = re.search(pattern, prefix, re.UNICODE)
826 826 assert token_match is not None # silence mypy
827 827 token_start = token_match.start()
828 828 token_prefix = token_match.group()
829 829
830 830 matched:List[str] = []
831 831 for key in filtered_keys:
832 832 try:
833 833 if not key.startswith(prefix_str):
834 834 continue
835 835 except (AttributeError, TypeError, UnicodeError):
836 836 # Python 3+ TypeError on b'a'.startswith('a') or vice-versa
837 837 continue
838 838
839 839 # reformat remainder of key to begin with prefix
840 840 rem = key[len(prefix_str):]
841 841 # force repr wrapped in '
842 842 rem_repr = repr(rem + '"') if isinstance(rem, str) else repr(rem + b'"')
843 843 rem_repr = rem_repr[1 + rem_repr.index("'"):-2]
844 844 if quote == '"':
845 845 # The entered prefix is quoted with ",
846 846 # but the match is quoted with '.
847 847 # A contained " hence needs escaping for comparison:
848 848 rem_repr = rem_repr.replace('"', '\\"')
849 849
850 850 # then reinsert prefix from start of token
851 851 matched.append('%s%s' % (token_prefix, rem_repr))
852 852 return quote, token_start, matched
853 853
854 854
855 855 def cursor_to_position(text:str, line:int, column:int)->int:
856 856 """
857 857 Convert the (line,column) position of the cursor in text to an offset in a
858 858 string.
859 859
860 860 Parameters
861 861 ----------
862 862 text : str
863 863 The text in which to calculate the cursor offset
864 864 line : int
865 865 Line of the cursor; 0-indexed
866 866 column : int
867 867 Column of the cursor 0-indexed
868 868
869 869 Returns
870 870 -------
871 871 Position of the cursor in ``text``, 0-indexed.
872 872
873 873 See Also
874 874 --------
875 875 position_to_cursor : reciprocal of this function
876 876
877 877 """
878 878 lines = text.split('\n')
879 879 assert line <= len(lines), '{} <= {}'.format(str(line), str(len(lines)))
880 880
881 881 return sum(len(l) + 1 for l in lines[:line]) + column
882 882
883 883 def position_to_cursor(text:str, offset:int)->Tuple[int, int]:
884 884 """
885 885 Convert the position of the cursor in text (0 indexed) to a line
886 886 number(0-indexed) and a column number (0-indexed) pair
887 887
888 888 Position should be a valid position in ``text``.
889 889
890 890 Parameters
891 891 ----------
892 892 text : str
893 893 The text in which to calculate the cursor offset
894 894 offset : int
895 895 Position of the cursor in ``text``, 0-indexed.
896 896
897 897 Returns
898 898 -------
899 899 (line, column) : (int, int)
900 900 Line of the cursor; 0-indexed, column of the cursor 0-indexed
901 901
902 902 See Also
903 903 --------
904 904 cursor_to_position : reciprocal of this function
905 905
906 906 """
907 907
908 908 assert 0 <= offset <= len(text) , "0 <= %s <= %s" % (offset , len(text))
909 909
910 910 before = text[:offset]
911 911 blines = before.split('\n') # ! splitnes trim trailing \n
912 912 line = before.count('\n')
913 913 col = len(blines[-1])
914 914 return line, col
915 915
916 916
917 917 def _safe_isinstance(obj, module, class_name):
918 918 """Checks if obj is an instance of module.class_name if loaded
919 919 """
920 920 return (module in sys.modules and
921 921 isinstance(obj, getattr(import_module(module), class_name)))
922 922
923 923 def back_unicode_name_matches(text:str) -> Tuple[str, Sequence[str]]:
924 924 """Match Unicode characters back to Unicode name
925 925
926 926 This does ``☃`` -> ``\\snowman``
927 927
928 928 Note that snowman is not a valid python3 combining character but will be expanded.
929 929 Though it will not recombine back to the snowman character by the completion machinery.
930 930
931 931 This will not either back-complete standard sequences like \\n, \\b ...
932 932
933 933 Returns
934 934 =======
935 935
936 936 Return a tuple with two elements:
937 937
938 938 - The Unicode character that was matched (preceded with a backslash), or
939 939 empty string,
940 940 - a sequence (of 1), name for the match Unicode character, preceded by
941 941 backslash, or empty if no match.
942 942
943 943 """
944 944 if len(text)<2:
945 945 return '', ()
946 946 maybe_slash = text[-2]
947 947 if maybe_slash != '\\':
948 948 return '', ()
949 949
950 950 char = text[-1]
951 951 # no expand on quote for completion in strings.
952 952 # nor backcomplete standard ascii keys
953 953 if char in string.ascii_letters or char in ('"',"'"):
954 954 return '', ()
955 955 try :
956 956 unic = unicodedata.name(char)
957 957 return '\\'+char,('\\'+unic,)
958 958 except KeyError:
959 959 pass
960 960 return '', ()
961 961
962 962 def back_latex_name_matches(text:str) -> Tuple[str, Sequence[str]] :
963 963 """Match latex characters back to unicode name
964 964
965 965 This does ``\\ℵ`` -> ``\\aleph``
966 966
967 967 """
968 968 if len(text)<2:
969 969 return '', ()
970 970 maybe_slash = text[-2]
971 971 if maybe_slash != '\\':
972 972 return '', ()
973 973
974 974
975 975 char = text[-1]
976 976 # no expand on quote for completion in strings.
977 977 # nor backcomplete standard ascii keys
978 978 if char in string.ascii_letters or char in ('"',"'"):
979 979 return '', ()
980 980 try :
981 981 latex = reverse_latex_symbol[char]
982 982 # '\\' replace the \ as well
983 983 return '\\'+char,[latex]
984 984 except KeyError:
985 985 pass
986 986 return '', ()
987 987
988 988
989 989 def _formatparamchildren(parameter) -> str:
990 990 """
991 991 Get parameter name and value from Jedi Private API
992 992
993 993 Jedi does not expose a simple way to get `param=value` from its API.
994 994
995 995 Parameters
996 996 ----------
997 997 parameter
998 998 Jedi's function `Param`
999 999
1000 1000 Returns
1001 1001 -------
1002 1002 A string like 'a', 'b=1', '*args', '**kwargs'
1003 1003
1004 1004 """
1005 1005 description = parameter.description
1006 1006 if not description.startswith('param '):
1007 1007 raise ValueError('Jedi function parameter description have change format.'
1008 1008 'Expected "param ...", found %r".' % description)
1009 1009 return description[6:]
1010 1010
1011 1011 def _make_signature(completion)-> str:
1012 1012 """
1013 1013 Make the signature from a jedi completion
1014 1014
1015 1015 Parameters
1016 1016 ----------
1017 1017 completion : jedi.Completion
1018 1018 object does not complete a function type
1019 1019
1020 1020 Returns
1021 1021 -------
1022 1022 a string consisting of the function signature, with the parenthesis but
1023 1023 without the function name. example:
1024 1024 `(a, *args, b=1, **kwargs)`
1025 1025
1026 1026 """
1027 1027
1028 1028 # it looks like this might work on jedi 0.17
1029 1029 if hasattr(completion, 'get_signatures'):
1030 1030 signatures = completion.get_signatures()
1031 1031 if not signatures:
1032 1032 return '(?)'
1033 1033
1034 1034 c0 = completion.get_signatures()[0]
1035 1035 return '('+c0.to_string().split('(', maxsplit=1)[1]
1036 1036
1037 1037 return '(%s)'% ', '.join([f for f in (_formatparamchildren(p) for signature in completion.get_signatures()
1038 1038 for p in signature.defined_names()) if f])
1039 1039
1040 1040
1041 1041 class _CompleteResult(NamedTuple):
1042 1042 matched_text : str
1043 1043 matches: Sequence[str]
1044 1044 matches_origin: Sequence[str]
1045 1045 jedi_matches: Any
1046 1046
1047 1047
1048 1048 class IPCompleter(Completer):
1049 1049 """Extension of the completer class with IPython-specific features"""
1050 1050
1051 1051 __dict_key_regexps: Optional[Dict[bool,Pattern]] = None
1052 1052
1053 1053 @observe('greedy')
1054 1054 def _greedy_changed(self, change):
1055 1055 """update the splitter and readline delims when greedy is changed"""
1056 1056 if change['new']:
1057 1057 self.splitter.delims = GREEDY_DELIMS
1058 1058 else:
1059 1059 self.splitter.delims = DELIMS
1060 1060
1061 1061 dict_keys_only = Bool(False,
1062 1062 help="""Whether to show dict key matches only""")
1063 1063
1064 1064 merge_completions = Bool(True,
1065 1065 help="""Whether to merge completion results into a single list
1066 1066
1067 1067 If False, only the completion results from the first non-empty
1068 1068 completer will be returned.
1069 1069 """
1070 1070 ).tag(config=True)
1071 1071 omit__names = Enum((0,1,2), default_value=2,
1072 1072 help="""Instruct the completer to omit private method names
1073 1073
1074 1074 Specifically, when completing on ``object.<tab>``.
1075 1075
1076 1076 When 2 [default]: all names that start with '_' will be excluded.
1077 1077
1078 1078 When 1: all 'magic' names (``__foo__``) will be excluded.
1079 1079
1080 1080 When 0: nothing will be excluded.
1081 1081 """
1082 1082 ).tag(config=True)
1083 1083 limit_to__all__ = Bool(False,
1084 1084 help="""
1085 1085 DEPRECATED as of version 5.0.
1086 1086
1087 1087 Instruct the completer to use __all__ for the completion
1088 1088
1089 1089 Specifically, when completing on ``object.<tab>``.
1090 1090
1091 1091 When True: only those names in obj.__all__ will be included.
1092 1092
1093 1093 When False [default]: the __all__ attribute is ignored
1094 1094 """,
1095 1095 ).tag(config=True)
1096 1096
1097 1097 profile_completions = Bool(
1098 1098 default_value=False,
1099 1099 help="If True, emit profiling data for completion subsystem using cProfile."
1100 1100 ).tag(config=True)
1101 1101
1102 1102 profiler_output_dir = Unicode(
1103 1103 default_value=".completion_profiles",
1104 1104 help="Template for path at which to output profile data for completions."
1105 1105 ).tag(config=True)
1106 1106
1107 1107 @observe('limit_to__all__')
1108 1108 def _limit_to_all_changed(self, change):
1109 1109 warnings.warn('`IPython.core.IPCompleter.limit_to__all__` configuration '
1110 1110 'value has been deprecated since IPython 5.0, will be made to have '
1111 1111 'no effects and then removed in future version of IPython.',
1112 1112 UserWarning)
1113 1113
1114 1114 def __init__(
1115 1115 self, shell=None, namespace=None, global_namespace=None, config=None, **kwargs
1116 1116 ):
1117 1117 """IPCompleter() -> completer
1118 1118
1119 1119 Return a completer object.
1120 1120
1121 1121 Parameters
1122 1122 ----------
1123 1123 shell
1124 1124 a pointer to the ipython shell itself. This is needed
1125 1125 because this completer knows about magic functions, and those can
1126 1126 only be accessed via the ipython instance.
1127 1127 namespace : dict, optional
1128 1128 an optional dict where completions are performed.
1129 1129 global_namespace : dict, optional
1130 1130 secondary optional dict for completions, to
1131 1131 handle cases (such as IPython embedded inside functions) where
1132 1132 both Python scopes are visible.
1133 1133 config : Config
1134 1134 traitlet's config object
1135 1135 **kwargs
1136 1136 passed to super class unmodified.
1137 1137 """
1138 1138
1139 1139 self.magic_escape = ESC_MAGIC
1140 1140 self.splitter = CompletionSplitter()
1141 1141
1142 1142 # _greedy_changed() depends on splitter and readline being defined:
1143 1143 super().__init__(
1144 1144 namespace=namespace,
1145 1145 global_namespace=global_namespace,
1146 1146 config=config,
1147 1147 **kwargs
1148 1148 )
1149 1149
1150 1150 # List where completion matches will be stored
1151 1151 self.matches = []
1152 1152 self.shell = shell
1153 1153 # Regexp to split filenames with spaces in them
1154 1154 self.space_name_re = re.compile(r'([^\\] )')
1155 1155 # Hold a local ref. to glob.glob for speed
1156 1156 self.glob = glob.glob
1157 1157
1158 1158 # Determine if we are running on 'dumb' terminals, like (X)Emacs
1159 1159 # buffers, to avoid completion problems.
1160 1160 term = os.environ.get('TERM','xterm')
1161 1161 self.dumb_terminal = term in ['dumb','emacs']
1162 1162
1163 1163 # Special handling of backslashes needed in win32 platforms
1164 1164 if sys.platform == "win32":
1165 1165 self.clean_glob = self._clean_glob_win32
1166 1166 else:
1167 1167 self.clean_glob = self._clean_glob
1168 1168
1169 1169 #regexp to parse docstring for function signature
1170 1170 self.docstring_sig_re = re.compile(r'^[\w|\s.]+\(([^)]*)\).*')
1171 1171 self.docstring_kwd_re = re.compile(r'[\s|\[]*(\w+)(?:\s*=\s*.*)')
1172 1172 #use this if positional argument name is also needed
1173 1173 #= re.compile(r'[\s|\[]*(\w+)(?:\s*=?\s*.*)')
1174 1174
1175 1175 self.magic_arg_matchers = [
1176 1176 self.magic_config_matches,
1177 1177 self.magic_color_matches,
1178 1178 ]
1179 1179
1180 1180 # This is set externally by InteractiveShell
1181 1181 self.custom_completers = None
1182 1182
1183 1183 # This is a list of names of unicode characters that can be completed
1184 1184 # into their corresponding unicode value. The list is large, so we
1185 1185 # lazily initialize it on first use. Consuming code should access this
1186 1186 # attribute through the `@unicode_names` property.
1187 1187 self._unicode_names = None
1188 1188
1189 1189 @property
1190 1190 def matchers(self) -> List[Any]:
1191 1191 """All active matcher routines for completion"""
1192 1192 if self.dict_keys_only:
1193 1193 return [self.dict_key_matches]
1194 1194
1195 1195 if self.use_jedi:
1196 1196 return [
1197 1197 *self.custom_matchers,
1198 1198 self.dict_key_matches,
1199 1199 self.file_matches,
1200 1200 self.magic_matches,
1201 1201 ]
1202 1202 else:
1203 1203 return [
1204 1204 *self.custom_matchers,
1205 1205 self.dict_key_matches,
1206 1206 self.python_matches,
1207 1207 self.file_matches,
1208 1208 self.magic_matches,
1209 1209 self.python_func_kw_matches,
1210 1210 ]
1211 1211
1212 1212 def all_completions(self, text:str) -> List[str]:
1213 1213 """
1214 1214 Wrapper around the completion methods for the benefit of emacs.
1215 1215 """
1216 1216 prefix = text.rpartition('.')[0]
1217 1217 with provisionalcompleter():
1218 1218 return ['.'.join([prefix, c.text]) if prefix and self.use_jedi else c.text
1219 1219 for c in self.completions(text, len(text))]
1220 1220
1221 1221 return self.complete(text)[1]
1222 1222
1223 1223 def _clean_glob(self, text:str):
1224 1224 return self.glob("%s*" % text)
1225 1225
1226 1226 def _clean_glob_win32(self, text:str):
1227 1227 return [f.replace("\\","/")
1228 1228 for f in self.glob("%s*" % text)]
1229 1229
1230 1230 def file_matches(self, text:str)->List[str]:
1231 1231 """Match filenames, expanding ~USER type strings.
1232 1232
1233 1233 Most of the seemingly convoluted logic in this completer is an
1234 1234 attempt to handle filenames with spaces in them. And yet it's not
1235 1235 quite perfect, because Python's readline doesn't expose all of the
1236 1236 GNU readline details needed for this to be done correctly.
1237 1237
1238 1238 For a filename with a space in it, the printed completions will be
1239 1239 only the parts after what's already been typed (instead of the
1240 1240 full completions, as is normally done). I don't think with the
1241 1241 current (as of Python 2.3) Python readline it's possible to do
1242 1242 better."""
1243 1243
1244 1244 # chars that require escaping with backslash - i.e. chars
1245 1245 # that readline treats incorrectly as delimiters, but we
1246 1246 # don't want to treat as delimiters in filename matching
1247 1247 # when escaped with backslash
1248 1248 if text.startswith('!'):
1249 1249 text = text[1:]
1250 1250 text_prefix = u'!'
1251 1251 else:
1252 1252 text_prefix = u''
1253 1253
1254 1254 text_until_cursor = self.text_until_cursor
1255 1255 # track strings with open quotes
1256 1256 open_quotes = has_open_quotes(text_until_cursor)
1257 1257
1258 1258 if '(' in text_until_cursor or '[' in text_until_cursor:
1259 1259 lsplit = text
1260 1260 else:
1261 1261 try:
1262 1262 # arg_split ~ shlex.split, but with unicode bugs fixed by us
1263 1263 lsplit = arg_split(text_until_cursor)[-1]
1264 1264 except ValueError:
1265 1265 # typically an unmatched ", or backslash without escaped char.
1266 1266 if open_quotes:
1267 1267 lsplit = text_until_cursor.split(open_quotes)[-1]
1268 1268 else:
1269 1269 return []
1270 1270 except IndexError:
1271 1271 # tab pressed on empty line
1272 1272 lsplit = ""
1273 1273
1274 1274 if not open_quotes and lsplit != protect_filename(lsplit):
1275 1275 # if protectables are found, do matching on the whole escaped name
1276 1276 has_protectables = True
1277 1277 text0,text = text,lsplit
1278 1278 else:
1279 1279 has_protectables = False
1280 1280 text = os.path.expanduser(text)
1281 1281
1282 1282 if text == "":
1283 1283 return [text_prefix + protect_filename(f) for f in self.glob("*")]
1284 1284
1285 1285 # Compute the matches from the filesystem
1286 1286 if sys.platform == 'win32':
1287 1287 m0 = self.clean_glob(text)
1288 1288 else:
1289 1289 m0 = self.clean_glob(text.replace('\\', ''))
1290 1290
1291 1291 if has_protectables:
1292 1292 # If we had protectables, we need to revert our changes to the
1293 1293 # beginning of filename so that we don't double-write the part
1294 1294 # of the filename we have so far
1295 1295 len_lsplit = len(lsplit)
1296 1296 matches = [text_prefix + text0 +
1297 1297 protect_filename(f[len_lsplit:]) for f in m0]
1298 1298 else:
1299 1299 if open_quotes:
1300 1300 # if we have a string with an open quote, we don't need to
1301 1301 # protect the names beyond the quote (and we _shouldn't_, as
1302 1302 # it would cause bugs when the filesystem call is made).
1303 1303 matches = m0 if sys.platform == "win32" else\
1304 1304 [protect_filename(f, open_quotes) for f in m0]
1305 1305 else:
1306 1306 matches = [text_prefix +
1307 1307 protect_filename(f) for f in m0]
1308 1308
1309 1309 # Mark directories in input list by appending '/' to their names.
1310 1310 return [x+'/' if os.path.isdir(x) else x for x in matches]
1311 1311
1312 1312 def magic_matches(self, text:str):
1313 1313 """Match magics"""
1314 1314 # Get all shell magics now rather than statically, so magics loaded at
1315 1315 # runtime show up too.
1316 1316 lsm = self.shell.magics_manager.lsmagic()
1317 1317 line_magics = lsm['line']
1318 1318 cell_magics = lsm['cell']
1319 1319 pre = self.magic_escape
1320 1320 pre2 = pre+pre
1321 1321
1322 1322 explicit_magic = text.startswith(pre)
1323 1323
1324 1324 # Completion logic:
1325 1325 # - user gives %%: only do cell magics
1326 1326 # - user gives %: do both line and cell magics
1327 1327 # - no prefix: do both
1328 1328 # In other words, line magics are skipped if the user gives %% explicitly
1329 1329 #
1330 1330 # We also exclude magics that match any currently visible names:
1331 1331 # https://github.com/ipython/ipython/issues/4877, unless the user has
1332 1332 # typed a %:
1333 1333 # https://github.com/ipython/ipython/issues/10754
1334 1334 bare_text = text.lstrip(pre)
1335 1335 global_matches = self.global_matches(bare_text)
1336 1336 if not explicit_magic:
1337 1337 def matches(magic):
1338 1338 """
1339 1339 Filter magics, in particular remove magics that match
1340 1340 a name present in global namespace.
1341 1341 """
1342 1342 return ( magic.startswith(bare_text) and
1343 1343 magic not in global_matches )
1344 1344 else:
1345 1345 def matches(magic):
1346 1346 return magic.startswith(bare_text)
1347 1347
1348 1348 comp = [ pre2+m for m in cell_magics if matches(m)]
1349 1349 if not text.startswith(pre2):
1350 1350 comp += [ pre+m for m in line_magics if matches(m)]
1351 1351
1352 1352 return comp
1353 1353
1354 1354 def magic_config_matches(self, text:str) -> List[str]:
1355 1355 """ Match class names and attributes for %config magic """
1356 1356 texts = text.strip().split()
1357 1357
1358 1358 if len(texts) > 0 and (texts[0] == 'config' or texts[0] == '%config'):
1359 1359 # get all configuration classes
1360 1360 classes = sorted(set([ c for c in self.shell.configurables
1361 1361 if c.__class__.class_traits(config=True)
1362 1362 ]), key=lambda x: x.__class__.__name__)
1363 1363 classnames = [ c.__class__.__name__ for c in classes ]
1364 1364
1365 1365 # return all classnames if config or %config is given
1366 1366 if len(texts) == 1:
1367 1367 return classnames
1368 1368
1369 1369 # match classname
1370 1370 classname_texts = texts[1].split('.')
1371 1371 classname = classname_texts[0]
1372 1372 classname_matches = [ c for c in classnames
1373 1373 if c.startswith(classname) ]
1374 1374
1375 1375 # return matched classes or the matched class with attributes
1376 1376 if texts[1].find('.') < 0:
1377 1377 return classname_matches
1378 1378 elif len(classname_matches) == 1 and \
1379 1379 classname_matches[0] == classname:
1380 1380 cls = classes[classnames.index(classname)].__class__
1381 1381 help = cls.class_get_help()
1382 1382 # strip leading '--' from cl-args:
1383 1383 help = re.sub(re.compile(r'^--', re.MULTILINE), '', help)
1384 1384 return [ attr.split('=')[0]
1385 1385 for attr in help.strip().splitlines()
1386 1386 if attr.startswith(texts[1]) ]
1387 1387 return []
1388 1388
1389 1389 def magic_color_matches(self, text:str) -> List[str] :
1390 1390 """ Match color schemes for %colors magic"""
1391 1391 texts = text.split()
1392 1392 if text.endswith(' '):
1393 1393 # .split() strips off the trailing whitespace. Add '' back
1394 1394 # so that: '%colors ' -> ['%colors', '']
1395 1395 texts.append('')
1396 1396
1397 1397 if len(texts) == 2 and (texts[0] == 'colors' or texts[0] == '%colors'):
1398 1398 prefix = texts[1]
1399 1399 return [ color for color in InspectColors.keys()
1400 1400 if color.startswith(prefix) ]
1401 1401 return []
1402 1402
1403 1403 def _jedi_matches(self, cursor_column:int, cursor_line:int, text:str) -> Iterable[Any]:
1404 1404 """
1405 1405 Return a list of :any:`jedi.api.Completions` object from a ``text`` and
1406 1406 cursor position.
1407 1407
1408 1408 Parameters
1409 1409 ----------
1410 1410 cursor_column : int
1411 1411 column position of the cursor in ``text``, 0-indexed.
1412 1412 cursor_line : int
1413 1413 line position of the cursor in ``text``, 0-indexed
1414 1414 text : str
1415 1415 text to complete
1416 1416
1417 1417 Notes
1418 1418 -----
1419 1419 If ``IPCompleter.debug`` is ``True`` may return a :any:`_FakeJediCompletion`
1420 1420 object containing a string with the Jedi debug information attached.
1421 1421 """
1422 1422 namespaces = [self.namespace]
1423 1423 if self.global_namespace is not None:
1424 1424 namespaces.append(self.global_namespace)
1425 1425
1426 1426 completion_filter = lambda x:x
1427 1427 offset = cursor_to_position(text, cursor_line, cursor_column)
1428 1428 # filter output if we are completing for object members
1429 1429 if offset:
1430 1430 pre = text[offset-1]
1431 1431 if pre == '.':
1432 1432 if self.omit__names == 2:
1433 1433 completion_filter = lambda c:not c.name.startswith('_')
1434 1434 elif self.omit__names == 1:
1435 1435 completion_filter = lambda c:not (c.name.startswith('__') and c.name.endswith('__'))
1436 1436 elif self.omit__names == 0:
1437 1437 completion_filter = lambda x:x
1438 1438 else:
1439 1439 raise ValueError("Don't understand self.omit__names == {}".format(self.omit__names))
1440 1440
1441 1441 interpreter = jedi.Interpreter(text[:offset], namespaces)
1442 1442 try_jedi = True
1443 1443
1444 1444 try:
1445 1445 # find the first token in the current tree -- if it is a ' or " then we are in a string
1446 1446 completing_string = False
1447 1447 try:
1448 1448 first_child = next(c for c in interpreter._get_module().tree_node.children if hasattr(c, 'value'))
1449 1449 except StopIteration:
1450 1450 pass
1451 1451 else:
1452 1452 # note the value may be ', ", or it may also be ''' or """, or
1453 1453 # in some cases, """what/you/typed..., but all of these are
1454 1454 # strings.
1455 1455 completing_string = len(first_child.value) > 0 and first_child.value[0] in {"'", '"'}
1456 1456
1457 1457 # if we are in a string jedi is likely not the right candidate for
1458 1458 # now. Skip it.
1459 1459 try_jedi = not completing_string
1460 1460 except Exception as e:
1461 1461 # many of things can go wrong, we are using private API just don't crash.
1462 1462 if self.debug:
1463 1463 print("Error detecting if completing a non-finished string :", e, '|')
1464 1464
1465 1465 if not try_jedi:
1466 1466 return []
1467 1467 try:
1468 1468 return filter(completion_filter, interpreter.complete(column=cursor_column, line=cursor_line + 1))
1469 1469 except Exception as e:
1470 1470 if self.debug:
1471 1471 return [_FakeJediCompletion('Oops Jedi has crashed, please report a bug with the following:\n"""\n%s\ns"""' % (e))]
1472 1472 else:
1473 1473 return []
1474 1474
1475 1475 def python_matches(self, text:str)->List[str]:
1476 1476 """Match attributes or global python names"""
1477 1477 if "." in text:
1478 1478 try:
1479 1479 matches = self.attr_matches(text)
1480 1480 if text.endswith('.') and self.omit__names:
1481 1481 if self.omit__names == 1:
1482 1482 # true if txt is _not_ a __ name, false otherwise:
1483 1483 no__name = (lambda txt:
1484 1484 re.match(r'.*\.__.*?__',txt) is None)
1485 1485 else:
1486 1486 # true if txt is _not_ a _ name, false otherwise:
1487 1487 no__name = (lambda txt:
1488 1488 re.match(r'\._.*?',txt[txt.rindex('.'):]) is None)
1489 1489 matches = filter(no__name, matches)
1490 1490 except NameError:
1491 1491 # catches <undefined attributes>.<tab>
1492 1492 matches = []
1493 1493 else:
1494 1494 matches = self.global_matches(text)
1495 1495 return matches
1496 1496
1497 1497 def _default_arguments_from_docstring(self, doc):
1498 1498 """Parse the first line of docstring for call signature.
1499 1499
1500 1500 Docstring should be of the form 'min(iterable[, key=func])\n'.
1501 1501 It can also parse cython docstring of the form
1502 1502 'Minuit.migrad(self, int ncall=10000, resume=True, int nsplit=1)'.
1503 1503 """
1504 1504 if doc is None:
1505 1505 return []
1506 1506
1507 1507 #care only the firstline
1508 1508 line = doc.lstrip().splitlines()[0]
1509 1509
1510 1510 #p = re.compile(r'^[\w|\s.]+\(([^)]*)\).*')
1511 1511 #'min(iterable[, key=func])\n' -> 'iterable[, key=func]'
1512 1512 sig = self.docstring_sig_re.search(line)
1513 1513 if sig is None:
1514 1514 return []
1515 1515 # iterable[, key=func]' -> ['iterable[' ,' key=func]']
1516 1516 sig = sig.groups()[0].split(',')
1517 1517 ret = []
1518 1518 for s in sig:
1519 1519 #re.compile(r'[\s|\[]*(\w+)(?:\s*=\s*.*)')
1520 1520 ret += self.docstring_kwd_re.findall(s)
1521 1521 return ret
1522 1522
1523 1523 def _default_arguments(self, obj):
1524 1524 """Return the list of default arguments of obj if it is callable,
1525 1525 or empty list otherwise."""
1526 1526 call_obj = obj
1527 1527 ret = []
1528 1528 if inspect.isbuiltin(obj):
1529 1529 pass
1530 1530 elif not (inspect.isfunction(obj) or inspect.ismethod(obj)):
1531 1531 if inspect.isclass(obj):
1532 1532 #for cython embedsignature=True the constructor docstring
1533 1533 #belongs to the object itself not __init__
1534 1534 ret += self._default_arguments_from_docstring(
1535 1535 getattr(obj, '__doc__', ''))
1536 1536 # for classes, check for __init__,__new__
1537 1537 call_obj = (getattr(obj, '__init__', None) or
1538 1538 getattr(obj, '__new__', None))
1539 1539 # for all others, check if they are __call__able
1540 1540 elif hasattr(obj, '__call__'):
1541 1541 call_obj = obj.__call__
1542 1542 ret += self._default_arguments_from_docstring(
1543 1543 getattr(call_obj, '__doc__', ''))
1544 1544
1545 1545 _keeps = (inspect.Parameter.KEYWORD_ONLY,
1546 1546 inspect.Parameter.POSITIONAL_OR_KEYWORD)
1547 1547
1548 1548 try:
1549 1549 sig = inspect.signature(obj)
1550 1550 ret.extend(k for k, v in sig.parameters.items() if
1551 1551 v.kind in _keeps)
1552 1552 except ValueError:
1553 1553 pass
1554 1554
1555 1555 return list(set(ret))
1556 1556
1557 1557 def python_func_kw_matches(self, text):
1558 1558 """Match named parameters (kwargs) of the last open function"""
1559 1559
1560 1560 if "." in text: # a parameter cannot be dotted
1561 1561 return []
1562 1562 try: regexp = self.__funcParamsRegex
1563 1563 except AttributeError:
1564 1564 regexp = self.__funcParamsRegex = re.compile(r'''
1565 1565 '.*?(?<!\\)' | # single quoted strings or
1566 1566 ".*?(?<!\\)" | # double quoted strings or
1567 1567 \w+ | # identifier
1568 1568 \S # other characters
1569 1569 ''', re.VERBOSE | re.DOTALL)
1570 1570 # 1. find the nearest identifier that comes before an unclosed
1571 1571 # parenthesis before the cursor
1572 1572 # e.g. for "foo (1+bar(x), pa<cursor>,a=1)", the candidate is "foo"
1573 1573 tokens = regexp.findall(self.text_until_cursor)
1574 1574 iterTokens = reversed(tokens); openPar = 0
1575 1575
1576 1576 for token in iterTokens:
1577 1577 if token == ')':
1578 1578 openPar -= 1
1579 1579 elif token == '(':
1580 1580 openPar += 1
1581 1581 if openPar > 0:
1582 1582 # found the last unclosed parenthesis
1583 1583 break
1584 1584 else:
1585 1585 return []
1586 1586 # 2. Concatenate dotted names ("foo.bar" for "foo.bar(x, pa" )
1587 1587 ids = []
1588 1588 isId = re.compile(r'\w+$').match
1589 1589
1590 1590 while True:
1591 1591 try:
1592 1592 ids.append(next(iterTokens))
1593 1593 if not isId(ids[-1]):
1594 1594 ids.pop(); break
1595 1595 if not next(iterTokens) == '.':
1596 1596 break
1597 1597 except StopIteration:
1598 1598 break
1599 1599
1600 1600 # Find all named arguments already assigned to, as to avoid suggesting
1601 1601 # them again
1602 1602 usedNamedArgs = set()
1603 1603 par_level = -1
1604 1604 for token, next_token in zip(tokens, tokens[1:]):
1605 1605 if token == '(':
1606 1606 par_level += 1
1607 1607 elif token == ')':
1608 1608 par_level -= 1
1609 1609
1610 1610 if par_level != 0:
1611 1611 continue
1612 1612
1613 1613 if next_token != '=':
1614 1614 continue
1615 1615
1616 1616 usedNamedArgs.add(token)
1617 1617
1618 1618 argMatches = []
1619 1619 try:
1620 1620 callableObj = '.'.join(ids[::-1])
1621 1621 namedArgs = self._default_arguments(eval(callableObj,
1622 1622 self.namespace))
1623 1623
1624 1624 # Remove used named arguments from the list, no need to show twice
1625 1625 for namedArg in set(namedArgs) - usedNamedArgs:
1626 1626 if namedArg.startswith(text):
1627 1627 argMatches.append("%s=" %namedArg)
1628 1628 except:
1629 1629 pass
1630 1630
1631 1631 return argMatches
1632 1632
1633 1633 @staticmethod
1634 1634 def _get_keys(obj: Any) -> List[Any]:
1635 1635 # Objects can define their own completions by defining an
1636 1636 # _ipy_key_completions_() method.
1637 1637 method = get_real_method(obj, '_ipython_key_completions_')
1638 1638 if method is not None:
1639 1639 return method()
1640 1640
1641 1641 # Special case some common in-memory dict-like types
1642 1642 if isinstance(obj, dict) or\
1643 1643 _safe_isinstance(obj, 'pandas', 'DataFrame'):
1644 1644 try:
1645 1645 return list(obj.keys())
1646 1646 except Exception:
1647 1647 return []
1648 1648 elif _safe_isinstance(obj, 'numpy', 'ndarray') or\
1649 1649 _safe_isinstance(obj, 'numpy', 'void'):
1650 1650 return obj.dtype.names or []
1651 1651 return []
1652 1652
1653 1653 def dict_key_matches(self, text:str) -> List[str]:
1654 1654 "Match string keys in a dictionary, after e.g. 'foo[' "
1655 1655
1656 1656
1657 1657 if self.__dict_key_regexps is not None:
1658 1658 regexps = self.__dict_key_regexps
1659 1659 else:
1660 1660 dict_key_re_fmt = r'''(?x)
1661 1661 ( # match dict-referring expression wrt greedy setting
1662 1662 %s
1663 1663 )
1664 1664 \[ # open bracket
1665 1665 \s* # and optional whitespace
1666 1666 # Capture any number of str-like objects (e.g. "a", "b", 'c')
1667 1667 ((?:[uUbB]? # string prefix (r not handled)
1668 1668 (?:
1669 1669 '(?:[^']|(?<!\\)\\')*'
1670 1670 |
1671 1671 "(?:[^"]|(?<!\\)\\")*"
1672 1672 )
1673 1673 \s*,\s*
1674 1674 )*)
1675 1675 ([uUbB]? # string prefix (r not handled)
1676 1676 (?: # unclosed string
1677 1677 '(?:[^']|(?<!\\)\\')*
1678 1678 |
1679 1679 "(?:[^"]|(?<!\\)\\")*
1680 1680 )
1681 1681 )?
1682 1682 $
1683 1683 '''
1684 1684 regexps = self.__dict_key_regexps = {
1685 1685 False: re.compile(dict_key_re_fmt % r'''
1686 1686 # identifiers separated by .
1687 1687 (?!\d)\w+
1688 1688 (?:\.(?!\d)\w+)*
1689 1689 '''),
1690 1690 True: re.compile(dict_key_re_fmt % '''
1691 1691 .+
1692 1692 ''')
1693 1693 }
1694 1694
1695 1695 match = regexps[self.greedy].search(self.text_until_cursor)
1696 1696
1697 1697 if match is None:
1698 1698 return []
1699 1699
1700 1700 expr, prefix0, prefix = match.groups()
1701 1701 try:
1702 1702 obj = eval(expr, self.namespace)
1703 1703 except Exception:
1704 1704 try:
1705 1705 obj = eval(expr, self.global_namespace)
1706 1706 except Exception:
1707 1707 return []
1708 1708
1709 1709 keys = self._get_keys(obj)
1710 1710 if not keys:
1711 1711 return keys
1712 1712
1713 1713 extra_prefix = eval(prefix0) if prefix0 != '' else None
1714 1714
1715 1715 closing_quote, token_offset, matches = match_dict_keys(keys, prefix, self.splitter.delims, extra_prefix=extra_prefix)
1716 1716 if not matches:
1717 1717 return matches
1718 1718
1719 1719 # get the cursor position of
1720 1720 # - the text being completed
1721 1721 # - the start of the key text
1722 1722 # - the start of the completion
1723 1723 text_start = len(self.text_until_cursor) - len(text)
1724 1724 if prefix:
1725 1725 key_start = match.start(3)
1726 1726 completion_start = key_start + token_offset
1727 1727 else:
1728 1728 key_start = completion_start = match.end()
1729 1729
1730 1730 # grab the leading prefix, to make sure all completions start with `text`
1731 1731 if text_start > key_start:
1732 1732 leading = ''
1733 1733 else:
1734 1734 leading = text[text_start:completion_start]
1735 1735
1736 1736 # the index of the `[` character
1737 1737 bracket_idx = match.end(1)
1738 1738
1739 1739 # append closing quote and bracket as appropriate
1740 1740 # this is *not* appropriate if the opening quote or bracket is outside
1741 1741 # the text given to this method
1742 1742 suf = ''
1743 1743 continuation = self.line_buffer[len(self.text_until_cursor):]
1744 1744 if key_start > text_start and closing_quote:
1745 1745 # quotes were opened inside text, maybe close them
1746 1746 if continuation.startswith(closing_quote):
1747 1747 continuation = continuation[len(closing_quote):]
1748 1748 else:
1749 1749 suf += closing_quote
1750 1750 if bracket_idx > text_start:
1751 1751 # brackets were opened inside text, maybe close them
1752 1752 if not continuation.startswith(']'):
1753 1753 suf += ']'
1754 1754
1755 1755 return [leading + k + suf for k in matches]
1756 1756
1757 1757 @staticmethod
1758 1758 def unicode_name_matches(text:str) -> Tuple[str, List[str]] :
1759 1759 """Match Latex-like syntax for unicode characters base
1760 1760 on the name of the character.
1761 1761
1762 1762 This does ``\\GREEK SMALL LETTER ETA`` -> ``η``
1763 1763
1764 1764 Works only on valid python 3 identifier, or on combining characters that
1765 1765 will combine to form a valid identifier.
1766 1766 """
1767 1767 slashpos = text.rfind('\\')
1768 1768 if slashpos > -1:
1769 1769 s = text[slashpos+1:]
1770 1770 try :
1771 1771 unic = unicodedata.lookup(s)
1772 1772 # allow combining chars
1773 1773 if ('a'+unic).isidentifier():
1774 1774 return '\\'+s,[unic]
1775 1775 except KeyError:
1776 1776 pass
1777 1777 return '', []
1778 1778
1779 1779
1780 1780 def latex_matches(self, text:str) -> Tuple[str, Sequence[str]]:
1781 1781 """Match Latex syntax for unicode characters.
1782 1782
1783 1783 This does both ``\\alp`` -> ``\\alpha`` and ``\\alpha`` -> ``α``
1784 1784 """
1785 1785 slashpos = text.rfind('\\')
1786 1786 if slashpos > -1:
1787 1787 s = text[slashpos:]
1788 1788 if s in latex_symbols:
1789 1789 # Try to complete a full latex symbol to unicode
1790 1790 # \\alpha -> α
1791 1791 return s, [latex_symbols[s]]
1792 1792 else:
1793 1793 # If a user has partially typed a latex symbol, give them
1794 1794 # a full list of options \al -> [\aleph, \alpha]
1795 1795 matches = [k for k in latex_symbols if k.startswith(s)]
1796 1796 if matches:
1797 1797 return s, matches
1798 1798 return '', ()
1799 1799
1800 1800 def dispatch_custom_completer(self, text):
1801 1801 if not self.custom_completers:
1802 1802 return
1803 1803
1804 1804 line = self.line_buffer
1805 1805 if not line.strip():
1806 1806 return None
1807 1807
1808 1808 # Create a little structure to pass all the relevant information about
1809 1809 # the current completion to any custom completer.
1810 1810 event = SimpleNamespace()
1811 1811 event.line = line
1812 1812 event.symbol = text
1813 1813 cmd = line.split(None,1)[0]
1814 1814 event.command = cmd
1815 1815 event.text_until_cursor = self.text_until_cursor
1816 1816
1817 1817 # for foo etc, try also to find completer for %foo
1818 1818 if not cmd.startswith(self.magic_escape):
1819 1819 try_magic = self.custom_completers.s_matches(
1820 1820 self.magic_escape + cmd)
1821 1821 else:
1822 1822 try_magic = []
1823 1823
1824 1824 for c in itertools.chain(self.custom_completers.s_matches(cmd),
1825 1825 try_magic,
1826 1826 self.custom_completers.flat_matches(self.text_until_cursor)):
1827 1827 try:
1828 1828 res = c(event)
1829 1829 if res:
1830 1830 # first, try case sensitive match
1831 1831 withcase = [r for r in res if r.startswith(text)]
1832 1832 if withcase:
1833 1833 return withcase
1834 1834 # if none, then case insensitive ones are ok too
1835 1835 text_low = text.lower()
1836 1836 return [r for r in res if r.lower().startswith(text_low)]
1837 1837 except TryNext:
1838 1838 pass
1839 1839 except KeyboardInterrupt:
1840 1840 """
1841 1841 If custom completer take too long,
1842 1842 let keyboard interrupt abort and return nothing.
1843 1843 """
1844 1844 break
1845 1845
1846 1846 return None
1847 1847
1848 1848 def completions(self, text: str, offset: int)->Iterator[Completion]:
1849 1849 """
1850 1850 Returns an iterator over the possible completions
1851 1851
1852 1852 .. warning::
1853 1853
1854 1854 Unstable
1855 1855
1856 1856 This function is unstable, API may change without warning.
1857 1857 It will also raise unless use in proper context manager.
1858 1858
1859 1859 Parameters
1860 1860 ----------
1861 1861 text : str
1862 1862 Full text of the current input, multi line string.
1863 1863 offset : int
1864 1864 Integer representing the position of the cursor in ``text``. Offset
1865 1865 is 0-based indexed.
1866 1866
1867 1867 Yields
1868 1868 ------
1869 1869 Completion
1870 1870
1871 1871 Notes
1872 1872 -----
1873 1873 The cursor on a text can either be seen as being "in between"
1874 1874 characters or "On" a character depending on the interface visible to
1875 1875 the user. For consistency the cursor being on "in between" characters X
1876 1876 and Y is equivalent to the cursor being "on" character Y, that is to say
1877 1877 the character the cursor is on is considered as being after the cursor.
1878 1878
1879 1879 Combining characters may span more that one position in the
1880 1880 text.
1881 1881
1882 1882 .. note::
1883 1883
1884 1884 If ``IPCompleter.debug`` is :any:`True` will yield a ``--jedi/ipython--``
1885 1885 fake Completion token to distinguish completion returned by Jedi
1886 1886 and usual IPython completion.
1887 1887
1888 1888 .. note::
1889 1889
1890 1890 Completions are not completely deduplicated yet. If identical
1891 1891 completions are coming from different sources this function does not
1892 1892 ensure that each completion object will only be present once.
1893 1893 """
1894 1894 warnings.warn("_complete is a provisional API (as of IPython 6.0). "
1895 1895 "It may change without warnings. "
1896 1896 "Use in corresponding context manager.",
1897 1897 category=ProvisionalCompleterWarning, stacklevel=2)
1898 1898
1899 1899 seen = set()
1900 1900 profiler:Optional[cProfile.Profile]
1901 1901 try:
1902 1902 if self.profile_completions:
1903 1903 import cProfile
1904 1904 profiler = cProfile.Profile()
1905 1905 profiler.enable()
1906 1906 else:
1907 1907 profiler = None
1908 1908
1909 1909 for c in self._completions(text, offset, _timeout=self.jedi_compute_type_timeout/1000):
1910 1910 if c and (c in seen):
1911 1911 continue
1912 1912 yield c
1913 1913 seen.add(c)
1914 1914 except KeyboardInterrupt:
1915 1915 """if completions take too long and users send keyboard interrupt,
1916 1916 do not crash and return ASAP. """
1917 1917 pass
1918 1918 finally:
1919 1919 if profiler is not None:
1920 1920 profiler.disable()
1921 1921 ensure_dir_exists(self.profiler_output_dir)
1922 1922 output_path = os.path.join(self.profiler_output_dir, str(uuid.uuid4()))
1923 1923 print("Writing profiler output to", output_path)
1924 1924 profiler.dump_stats(output_path)
1925 1925
1926 1926 def _completions(self, full_text: str, offset: int, *, _timeout) -> Iterator[Completion]:
1927 1927 """
1928 1928 Core completion module.Same signature as :any:`completions`, with the
1929 1929 extra `timeout` parameter (in seconds).
1930 1930
1931 1931 Computing jedi's completion ``.type`` can be quite expensive (it is a
1932 1932 lazy property) and can require some warm-up, more warm up than just
1933 1933 computing the ``name`` of a completion. The warm-up can be :
1934 1934
1935 1935 - Long warm-up the first time a module is encountered after
1936 1936 install/update: actually build parse/inference tree.
1937 1937
1938 1938 - first time the module is encountered in a session: load tree from
1939 1939 disk.
1940 1940
1941 1941 We don't want to block completions for tens of seconds so we give the
1942 1942 completer a "budget" of ``_timeout`` seconds per invocation to compute
1943 1943 completions types, the completions that have not yet been computed will
1944 1944 be marked as "unknown" an will have a chance to be computed next round
1945 1945 are things get cached.
1946 1946
1947 1947 Keep in mind that Jedi is not the only thing treating the completion so
1948 1948 keep the timeout short-ish as if we take more than 0.3 second we still
1949 1949 have lots of processing to do.
1950 1950
1951 1951 """
1952 1952 deadline = time.monotonic() + _timeout
1953 1953
1954 1954
1955 1955 before = full_text[:offset]
1956 1956 cursor_line, cursor_column = position_to_cursor(full_text, offset)
1957 1957
1958 1958 matched_text, matches, matches_origin, jedi_matches = self._complete(
1959 1959 full_text=full_text, cursor_line=cursor_line, cursor_pos=cursor_column)
1960 1960
1961 1961 iter_jm = iter(jedi_matches)
1962 1962 if _timeout:
1963 1963 for jm in iter_jm:
1964 1964 try:
1965 1965 type_ = jm.type
1966 1966 except Exception:
1967 1967 if self.debug:
1968 1968 print("Error in Jedi getting type of ", jm)
1969 1969 type_ = None
1970 1970 delta = len(jm.name_with_symbols) - len(jm.complete)
1971 1971 if type_ == 'function':
1972 1972 signature = _make_signature(jm)
1973 1973 else:
1974 1974 signature = ''
1975 1975 yield Completion(start=offset - delta,
1976 1976 end=offset,
1977 1977 text=jm.name_with_symbols,
1978 1978 type=type_,
1979 1979 signature=signature,
1980 1980 _origin='jedi')
1981 1981
1982 1982 if time.monotonic() > deadline:
1983 1983 break
1984 1984
1985 1985 for jm in iter_jm:
1986 1986 delta = len(jm.name_with_symbols) - len(jm.complete)
1987 1987 yield Completion(start=offset - delta,
1988 1988 end=offset,
1989 1989 text=jm.name_with_symbols,
1990 1990 type='<unknown>', # don't compute type for speed
1991 1991 _origin='jedi',
1992 1992 signature='')
1993 1993
1994 1994
1995 1995 start_offset = before.rfind(matched_text)
1996 1996
1997 1997 # TODO:
1998 1998 # Suppress this, right now just for debug.
1999 1999 if jedi_matches and matches and self.debug:
2000 2000 yield Completion(start=start_offset, end=offset, text='--jedi/ipython--',
2001 2001 _origin='debug', type='none', signature='')
2002 2002
2003 2003 # I'm unsure if this is always true, so let's assert and see if it
2004 2004 # crash
2005 2005 assert before.endswith(matched_text)
2006 2006 for m, t in zip(matches, matches_origin):
2007 2007 yield Completion(start=start_offset, end=offset, text=m, _origin=t, signature='', type='<unknown>')
2008 2008
2009 2009
2010 2010 def complete(self, text=None, line_buffer=None, cursor_pos=None) -> Tuple[str, Sequence[str]]:
2011 2011 """Find completions for the given text and line context.
2012 2012
2013 2013 Note that both the text and the line_buffer are optional, but at least
2014 2014 one of them must be given.
2015 2015
2016 2016 Parameters
2017 2017 ----------
2018 2018 text : string, optional
2019 2019 Text to perform the completion on. If not given, the line buffer
2020 2020 is split using the instance's CompletionSplitter object.
2021 2021 line_buffer : string, optional
2022 2022 If not given, the completer attempts to obtain the current line
2023 2023 buffer via readline. This keyword allows clients which are
2024 2024 requesting for text completions in non-readline contexts to inform
2025 2025 the completer of the entire text.
2026 2026 cursor_pos : int, optional
2027 2027 Index of the cursor in the full line buffer. Should be provided by
2028 2028 remote frontends where kernel has no access to frontend state.
2029 2029
2030 2030 Returns
2031 2031 -------
2032 2032 Tuple of two items:
2033 2033 text : str
2034 2034 Text that was actually used in the completion.
2035 2035 matches : list
2036 2036 A list of completion matches.
2037 2037
2038 2038 Notes
2039 2039 -----
2040 2040 This API is likely to be deprecated and replaced by
2041 2041 :any:`IPCompleter.completions` in the future.
2042 2042
2043 2043 """
2044 2044 warnings.warn('`Completer.complete` is pending deprecation since '
2045 2045 'IPython 6.0 and will be replaced by `Completer.completions`.',
2046 2046 PendingDeprecationWarning)
2047 2047 # potential todo, FOLD the 3rd throw away argument of _complete
2048 2048 # into the first 2 one.
2049 2049 return self._complete(line_buffer=line_buffer, cursor_pos=cursor_pos, text=text, cursor_line=0)[:2]
2050 2050
2051 2051 def _complete(self, *, cursor_line, cursor_pos, line_buffer=None, text=None,
2052 2052 full_text=None) -> _CompleteResult:
2053 2053 """
2054 2054 Like complete but can also returns raw jedi completions as well as the
2055 2055 origin of the completion text. This could (and should) be made much
2056 2056 cleaner but that will be simpler once we drop the old (and stateful)
2057 2057 :any:`complete` API.
2058 2058
2059 2059 With current provisional API, cursor_pos act both (depending on the
2060 2060 caller) as the offset in the ``text`` or ``line_buffer``, or as the
2061 2061 ``column`` when passing multiline strings this could/should be renamed
2062 2062 but would add extra noise.
2063 2063
2064 2064 Parameters
2065 2065 ----------
2066 2066 cursor_line
2067 2067 Index of the line the cursor is on. 0 indexed.
2068 2068 cursor_pos
2069 2069 Position of the cursor in the current line/line_buffer/text. 0
2070 2070 indexed.
2071 2071 line_buffer : optional, str
2072 2072 The current line the cursor is in, this is mostly due to legacy
2073 2073 reason that readline could only give a us the single current line.
2074 2074 Prefer `full_text`.
2075 2075 text : str
2076 2076 The current "token" the cursor is in, mostly also for historical
2077 2077 reasons. as the completer would trigger only after the current line
2078 2078 was parsed.
2079 2079 full_text : str
2080 2080 Full text of the current cell.
2081 2081
2082 2082 Returns
2083 2083 -------
2084 2084 A tuple of N elements which are (likely):
2085 2085 matched_text: ? the text that the complete matched
2086 2086 matches: list of completions ?
2087 2087 matches_origin: ? list same length as matches, and where each completion came from
2088 2088 jedi_matches: list of Jedi matches, have it's own structure.
2089 2089 """
2090 2090
2091 2091
2092 2092 # if the cursor position isn't given, the only sane assumption we can
2093 2093 # make is that it's at the end of the line (the common case)
2094 2094 if cursor_pos is None:
2095 2095 cursor_pos = len(line_buffer) if text is None else len(text)
2096 2096
2097 2097 if self.use_main_ns:
2098 2098 self.namespace = __main__.__dict__
2099 2099
2100 2100 # if text is either None or an empty string, rely on the line buffer
2101 2101 if (not line_buffer) and full_text:
2102 2102 line_buffer = full_text.split('\n')[cursor_line]
2103 2103 if not text: # issue #11508: check line_buffer before calling split_line
2104 2104 text = self.splitter.split_line(line_buffer, cursor_pos) if line_buffer else ''
2105 2105
2106 2106 if self.backslash_combining_completions:
2107 2107 # allow deactivation of these on windows.
2108 2108 base_text = text if not line_buffer else line_buffer[:cursor_pos]
2109 2109
2110 2110 for meth in (self.latex_matches,
2111 2111 self.unicode_name_matches,
2112 2112 back_latex_name_matches,
2113 2113 back_unicode_name_matches,
2114 2114 self.fwd_unicode_match):
2115 2115 name_text, name_matches = meth(base_text)
2116 2116 if name_text:
2117 2117 return _CompleteResult(name_text, name_matches[:MATCHES_LIMIT], \
2118 2118 [meth.__qualname__]*min(len(name_matches), MATCHES_LIMIT), ())
2119 2119
2120 2120
2121 2121 # If no line buffer is given, assume the input text is all there was
2122 2122 if line_buffer is None:
2123 2123 line_buffer = text
2124 2124
2125 2125 self.line_buffer = line_buffer
2126 2126 self.text_until_cursor = self.line_buffer[:cursor_pos]
2127 2127
2128 2128 # Do magic arg matches
2129 2129 for matcher in self.magic_arg_matchers:
2130 2130 matches = list(matcher(line_buffer))[:MATCHES_LIMIT]
2131 2131 if matches:
2132 2132 origins = [matcher.__qualname__] * len(matches)
2133 2133 return _CompleteResult(text, matches, origins, ())
2134 2134
2135 2135 # Start with a clean slate of completions
2136 2136 matches = []
2137 2137
2138 2138 # FIXME: we should extend our api to return a dict with completions for
2139 2139 # different types of objects. The rlcomplete() method could then
2140 2140 # simply collapse the dict into a list for readline, but we'd have
2141 2141 # richer completion semantics in other environments.
2142 2142 is_magic_prefix = len(text) > 0 and text[0] == "%"
2143 2143 completions: Iterable[Any] = []
2144 2144 if self.use_jedi and not is_magic_prefix:
2145 2145 if not full_text:
2146 2146 full_text = line_buffer
2147 2147 completions = self._jedi_matches(
2148 2148 cursor_pos, cursor_line, full_text)
2149 2149
2150 2150 if self.merge_completions:
2151 2151 matches = []
2152 2152 for matcher in self.matchers:
2153 2153 try:
2154 2154 matches.extend([(m, matcher.__qualname__)
2155 2155 for m in matcher(text)])
2156 2156 except:
2157 2157 # Show the ugly traceback if the matcher causes an
2158 2158 # exception, but do NOT crash the kernel!
2159 2159 sys.excepthook(*sys.exc_info())
2160 2160 else:
2161 2161 for matcher in self.matchers:
2162 2162 matches = [(m, matcher.__qualname__)
2163 2163 for m in matcher(text)]
2164 2164 if matches:
2165 2165 break
2166 2166
2167 2167 seen = set()
2168 2168 filtered_matches = set()
2169 2169 for m in matches:
2170 2170 t, c = m
2171 2171 if t not in seen:
2172 2172 filtered_matches.add(m)
2173 2173 seen.add(t)
2174 2174
2175 2175 _filtered_matches = sorted(filtered_matches, key=lambda x: completions_sorting_key(x[0]))
2176 2176
2177 2177 custom_res = [(m, 'custom') for m in self.dispatch_custom_completer(text) or []]
2178 2178
2179 2179 _filtered_matches = custom_res or _filtered_matches
2180 2180
2181 2181 _filtered_matches = _filtered_matches[:MATCHES_LIMIT]
2182 2182 _matches = [m[0] for m in _filtered_matches]
2183 2183 origins = [m[1] for m in _filtered_matches]
2184 2184
2185 2185 self.matches = _matches
2186 2186
2187 2187 return _CompleteResult(text, _matches, origins, completions)
2188 2188
2189 2189 def fwd_unicode_match(self, text:str) -> Tuple[str, Sequence[str]]:
2190 2190 """
2191 2191 Forward match a string starting with a backslash with a list of
2192 2192 potential Unicode completions.
2193 2193
2194 2194 Will compute list list of Unicode character names on first call and cache it.
2195 2195
2196 2196 Returns
2197 2197 -------
2198 2198 At tuple with:
2199 2199 - matched text (empty if no matches)
2200 2200 - list of potential completions, empty tuple otherwise)
2201 2201 """
2202 2202 # TODO: self.unicode_names is here a list we traverse each time with ~100k elements.
2203 2203 # We could do a faster match using a Trie.
2204 2204
2205 2205 # Using pygtrie the following seem to work:
2206 2206
2207 2207 # s = PrefixSet()
2208 2208
2209 2209 # for c in range(0,0x10FFFF + 1):
2210 2210 # try:
2211 2211 # s.add(unicodedata.name(chr(c)))
2212 2212 # except ValueError:
2213 2213 # pass
2214 2214 # [''.join(k) for k in s.iter(prefix)]
2215 2215
2216 2216 # But need to be timed and adds an extra dependency.
2217 2217
2218 2218 slashpos = text.rfind('\\')
2219 2219 # if text starts with slash
2220 2220 if slashpos > -1:
2221 2221 # PERF: It's important that we don't access self._unicode_names
2222 2222 # until we're inside this if-block. _unicode_names is lazily
2223 2223 # initialized, and it takes a user-noticeable amount of time to
2224 2224 # initialize it, so we don't want to initialize it unless we're
2225 2225 # actually going to use it.
2226 2226 s = text[slashpos + 1 :]
2227 2227 sup = s.upper()
2228 2228 candidates = [x for x in self.unicode_names if x.startswith(sup)]
2229 2229 if candidates:
2230 2230 return s, candidates
2231 2231 candidates = [x for x in self.unicode_names if sup in x]
2232 2232 if candidates:
2233 2233 return s, candidates
2234 2234 splitsup = sup.split(" ")
2235 2235 candidates = [
2236 2236 x for x in self.unicode_names if all(u in x for u in splitsup)
2237 2237 ]
2238 2238 if candidates:
2239 2239 return s, candidates
2240 2240
2241 2241 return "", ()
2242 2242
2243 2243 # if text does not start with slash
2244 2244 else:
2245 2245 return '', ()
2246 2246
2247 2247 @property
2248 2248 def unicode_names(self) -> List[str]:
2249 2249 """List of names of unicode code points that can be completed.
2250 2250
2251 2251 The list is lazily initialized on first access.
2252 2252 """
2253 2253 if self._unicode_names is None:
2254 2254 names = []
2255 2255 for c in range(0,0x10FFFF + 1):
2256 2256 try:
2257 2257 names.append(unicodedata.name(chr(c)))
2258 2258 except ValueError:
2259 2259 pass
2260 2260 self._unicode_names = _unicode_name_compute(_UNICODE_RANGES)
2261 2261
2262 2262 return self._unicode_names
2263 2263
2264 2264 def _unicode_name_compute(ranges:List[Tuple[int,int]]) -> List[str]:
2265 2265 names = []
2266 2266 for start,stop in ranges:
2267 2267 for c in range(start, stop) :
2268 2268 try:
2269 2269 names.append(unicodedata.name(chr(c)))
2270 2270 except ValueError:
2271 2271 pass
2272 2272 return names
@@ -1,913 +1,913 b''
1 1 """ History related magics and functionality """
2 2
3 3 # Copyright (c) IPython Development Team.
4 4 # Distributed under the terms of the Modified BSD License.
5 5
6 6
7 7 import atexit
8 8 import datetime
9 9 from pathlib import Path
10 10 import re
11 11 import sqlite3
12 12 import threading
13 13
14 14 from traitlets.config.configurable import LoggingConfigurable
15 15 from decorator import decorator
16 16 from IPython.utils.decorators import undoc
17 17 from IPython.paths import locate_profile
18 18 from traitlets import (
19 19 Any,
20 20 Bool,
21 21 Dict,
22 22 Instance,
23 23 Integer,
24 24 List,
25 25 Unicode,
26 26 Union,
27 27 TraitError,
28 28 default,
29 29 observe,
30 30 )
31 31
32 32 #-----------------------------------------------------------------------------
33 33 # Classes and functions
34 34 #-----------------------------------------------------------------------------
35 35
36 36 @undoc
37 37 class DummyDB(object):
38 38 """Dummy DB that will act as a black hole for history.
39 39
40 40 Only used in the absence of sqlite"""
41 41 def execute(*args, **kwargs):
42 42 return []
43 43
44 44 def commit(self, *args, **kwargs):
45 45 pass
46 46
47 47 def __enter__(self, *args, **kwargs):
48 48 pass
49 49
50 50 def __exit__(self, *args, **kwargs):
51 51 pass
52 52
53 53
54 54 @decorator
55 55 def only_when_enabled(f, self, *a, **kw):
56 56 """Decorator: return an empty list in the absence of sqlite."""
57 57 if not self.enabled:
58 58 return []
59 59 else:
60 60 return f(self, *a, **kw)
61 61
62 62
63 63 # use 16kB as threshold for whether a corrupt history db should be saved
64 64 # that should be at least 100 entries or so
65 65 _SAVE_DB_SIZE = 16384
66 66
67 67 @decorator
68 68 def catch_corrupt_db(f, self, *a, **kw):
69 69 """A decorator which wraps HistoryAccessor method calls to catch errors from
70 70 a corrupt SQLite database, move the old database out of the way, and create
71 71 a new one.
72 72
73 73 We avoid clobbering larger databases because this may be triggered due to filesystem issues,
74 74 not just a corrupt file.
75 75 """
76 76 try:
77 77 return f(self, *a, **kw)
78 78 except (sqlite3.DatabaseError, sqlite3.OperationalError) as e:
79 79 self._corrupt_db_counter += 1
80 80 self.log.error("Failed to open SQLite history %s (%s).", self.hist_file, e)
81 81 if self.hist_file != ':memory:':
82 82 if self._corrupt_db_counter > self._corrupt_db_limit:
83 83 self.hist_file = ':memory:'
84 84 self.log.error("Failed to load history too many times, history will not be saved.")
85 85 elif self.hist_file.is_file():
86 86 # move the file out of the way
87 87 base = str(self.hist_file.parent / self.hist_file.stem)
88 88 ext = self.hist_file.suffix
89 89 size = self.hist_file.stat().st_size
90 90 if size >= _SAVE_DB_SIZE:
91 91 # if there's significant content, avoid clobbering
92 92 now = datetime.datetime.now().isoformat().replace(':', '.')
93 93 newpath = base + '-corrupt-' + now + ext
94 94 # don't clobber previous corrupt backups
95 95 for i in range(100):
96 96 if not Path(newpath).exists():
97 97 break
98 98 else:
99 99 newpath = base + '-corrupt-' + now + (u'-%i' % i) + ext
100 100 else:
101 101 # not much content, possibly empty; don't worry about clobbering
102 102 # maybe we should just delete it?
103 103 newpath = base + '-corrupt' + ext
104 104 self.hist_file.rename(newpath)
105 105 self.log.error("History file was moved to %s and a new file created.", newpath)
106 106 self.init_db()
107 107 return []
108 108 else:
109 109 # Failed with :memory:, something serious is wrong
110 110 raise
111 111
112 112
113 113 class HistoryAccessorBase(LoggingConfigurable):
114 114 """An abstract class for History Accessors """
115 115
116 116 def get_tail(self, n=10, raw=True, output=False, include_latest=False):
117 117 raise NotImplementedError
118 118
119 119 def search(self, pattern="*", raw=True, search_raw=True,
120 120 output=False, n=None, unique=False):
121 121 raise NotImplementedError
122 122
123 123 def get_range(self, session, start=1, stop=None, raw=True,output=False):
124 124 raise NotImplementedError
125 125
126 126 def get_range_by_str(self, rangestr, raw=True, output=False):
127 127 raise NotImplementedError
128 128
129 129
130 130 class HistoryAccessor(HistoryAccessorBase):
131 131 """Access the history database without adding to it.
132 132
133 133 This is intended for use by standalone history tools. IPython shells use
134 134 HistoryManager, below, which is a subclass of this."""
135 135
136 136 # counter for init_db retries, so we don't keep trying over and over
137 137 _corrupt_db_counter = 0
138 138 # after two failures, fallback on :memory:
139 139 _corrupt_db_limit = 2
140 140
141 141 # String holding the path to the history file
142 142 hist_file = Union(
143 143 [Instance(Path), Unicode()],
144 144 help="""Path to file to use for SQLite history database.
145 145
146 146 By default, IPython will put the history database in the IPython
147 147 profile directory. If you would rather share one history among
148 148 profiles, you can set this value in each, so that they are consistent.
149 149
150 150 Due to an issue with fcntl, SQLite is known to misbehave on some NFS
151 151 mounts. If you see IPython hanging, try setting this to something on a
152 152 local disk, e.g::
153 153
154 154 ipython --HistoryManager.hist_file=/tmp/ipython_hist.sqlite
155 155
156 156 you can also use the specific value `:memory:` (including the colon
157 157 at both end but not the back ticks), to avoid creating an history file.
158 158
159 159 """,
160 160 ).tag(config=True)
161 161
162 162 enabled = Bool(True,
163 163 help="""enable the SQLite history
164 164
165 165 set enabled=False to disable the SQLite history,
166 166 in which case there will be no stored history, no SQLite connection,
167 167 and no background saving thread. This may be necessary in some
168 168 threaded environments where IPython is embedded.
169 """
169 """,
170 170 ).tag(config=True)
171 171
172 172 connection_options = Dict(
173 173 help="""Options for configuring the SQLite connection
174 174
175 175 These options are passed as keyword args to sqlite3.connect
176 176 when establishing database connections.
177 177 """
178 178 ).tag(config=True)
179 179
180 180 # The SQLite database
181 181 db = Any()
182 182 @observe('db')
183 183 def _db_changed(self, change):
184 184 """validate the db, since it can be an Instance of two different types"""
185 185 new = change['new']
186 186 connection_types = (DummyDB, sqlite3.Connection)
187 187 if not isinstance(new, connection_types):
188 188 msg = "%s.db must be sqlite3 Connection or DummyDB, not %r" % \
189 189 (self.__class__.__name__, new)
190 190 raise TraitError(msg)
191 191
192 192 def __init__(self, profile="default", hist_file="", **traits):
193 193 """Create a new history accessor.
194 194
195 195 Parameters
196 196 ----------
197 197 profile : str
198 198 The name of the profile from which to open history.
199 199 hist_file : str
200 200 Path to an SQLite history database stored by IPython. If specified,
201 201 hist_file overrides profile.
202 202 config : :class:`~traitlets.config.loader.Config`
203 203 Config object. hist_file can also be set through this.
204 204 """
205 205 # We need a pointer back to the shell for various tasks.
206 206 super(HistoryAccessor, self).__init__(**traits)
207 207 # defer setting hist_file from kwarg until after init,
208 208 # otherwise the default kwarg value would clobber any value
209 209 # set by config
210 210 if hist_file:
211 211 self.hist_file = hist_file
212 212
213 213 try:
214 214 self.hist_file
215 215 except TraitError:
216 216 # No one has set the hist_file, yet.
217 217 self.hist_file = self._get_hist_file_name(profile)
218 218
219 219 self.init_db()
220 220
221 221 def _get_hist_file_name(self, profile='default'):
222 222 """Find the history file for the given profile name.
223 223
224 224 This is overridden by the HistoryManager subclass, to use the shell's
225 225 active profile.
226 226
227 227 Parameters
228 228 ----------
229 229 profile : str
230 230 The name of a profile which has a history file.
231 231 """
232 232 return Path(locate_profile(profile)) / "history.sqlite"
233 233
234 234 @catch_corrupt_db
235 235 def init_db(self):
236 236 """Connect to the database, and create tables if necessary."""
237 237 if not self.enabled:
238 238 self.db = DummyDB()
239 239 return
240 240
241 241 # use detect_types so that timestamps return datetime objects
242 242 kwargs = dict(detect_types=sqlite3.PARSE_DECLTYPES|sqlite3.PARSE_COLNAMES)
243 243 kwargs.update(self.connection_options)
244 244 self.db = sqlite3.connect(str(self.hist_file), **kwargs)
245 245 with self.db:
246 246 self.db.execute(
247 247 """CREATE TABLE IF NOT EXISTS sessions (session integer
248 248 primary key autoincrement, start timestamp,
249 249 end timestamp, num_cmds integer, remark text)"""
250 250 )
251 251 self.db.execute(
252 252 """CREATE TABLE IF NOT EXISTS history
253 253 (session integer, line integer, source text, source_raw text,
254 254 PRIMARY KEY (session, line))"""
255 255 )
256 256 # Output history is optional, but ensure the table's there so it can be
257 257 # enabled later.
258 258 self.db.execute(
259 259 """CREATE TABLE IF NOT EXISTS output_history
260 260 (session integer, line integer, output text,
261 261 PRIMARY KEY (session, line))"""
262 262 )
263 263 # success! reset corrupt db count
264 264 self._corrupt_db_counter = 0
265 265
266 266 def writeout_cache(self):
267 267 """Overridden by HistoryManager to dump the cache before certain
268 268 database lookups."""
269 269 pass
270 270
271 271 ## -------------------------------
272 272 ## Methods for retrieving history:
273 273 ## -------------------------------
274 274 def _run_sql(self, sql, params, raw=True, output=False, latest=False):
275 275 """Prepares and runs an SQL query for the history database.
276 276
277 277 Parameters
278 278 ----------
279 279 sql : str
280 280 Any filtering expressions to go after SELECT ... FROM ...
281 281 params : tuple
282 282 Parameters passed to the SQL query (to replace "?")
283 283 raw, output : bool
284 284 See :meth:`get_range`
285 285 latest : bool
286 286 Select rows with max (session, line)
287 287
288 288 Returns
289 289 -------
290 290 Tuples as :meth:`get_range`
291 291 """
292 292 toget = 'source_raw' if raw else 'source'
293 293 sqlfrom = "history"
294 294 if output:
295 295 sqlfrom = "history LEFT JOIN output_history USING (session, line)"
296 296 toget = "history.%s, output_history.output" % toget
297 297 if latest:
298 298 toget += ", MAX(session * 128 * 1024 + line)"
299 299 cur = self.db.execute("SELECT session, line, %s FROM %s " %\
300 300 (toget, sqlfrom) + sql, params)
301 301 if latest:
302 302 cur = (row[:-1] for row in cur)
303 303 if output: # Regroup into 3-tuples, and parse JSON
304 304 return ((ses, lin, (inp, out)) for ses, lin, inp, out in cur)
305 305 return cur
306 306
307 307 @only_when_enabled
308 308 @catch_corrupt_db
309 309 def get_session_info(self, session):
310 310 """Get info about a session.
311 311
312 312 Parameters
313 313 ----------
314 314 session : int
315 315 Session number to retrieve.
316 316
317 317 Returns
318 318 -------
319 319 session_id : int
320 320 Session ID number
321 321 start : datetime
322 322 Timestamp for the start of the session.
323 323 end : datetime
324 324 Timestamp for the end of the session, or None if IPython crashed.
325 325 num_cmds : int
326 326 Number of commands run, or None if IPython crashed.
327 327 remark : unicode
328 328 A manually set description.
329 329 """
330 330 query = "SELECT * from sessions where session == ?"
331 331 return self.db.execute(query, (session,)).fetchone()
332 332
333 333 @catch_corrupt_db
334 334 def get_last_session_id(self):
335 335 """Get the last session ID currently in the database.
336 336
337 337 Within IPython, this should be the same as the value stored in
338 338 :attr:`HistoryManager.session_number`.
339 339 """
340 340 for record in self.get_tail(n=1, include_latest=True):
341 341 return record[0]
342 342
343 343 @catch_corrupt_db
344 344 def get_tail(self, n=10, raw=True, output=False, include_latest=False):
345 345 """Get the last n lines from the history database.
346 346
347 347 Parameters
348 348 ----------
349 349 n : int
350 350 The number of lines to get
351 351 raw, output : bool
352 352 See :meth:`get_range`
353 353 include_latest : bool
354 354 If False (default), n+1 lines are fetched, and the latest one
355 355 is discarded. This is intended to be used where the function
356 356 is called by a user command, which it should not return.
357 357
358 358 Returns
359 359 -------
360 360 Tuples as :meth:`get_range`
361 361 """
362 362 self.writeout_cache()
363 363 if not include_latest:
364 364 n += 1
365 365 cur = self._run_sql("ORDER BY session DESC, line DESC LIMIT ?",
366 366 (n,), raw=raw, output=output)
367 367 if not include_latest:
368 368 return reversed(list(cur)[1:])
369 369 return reversed(list(cur))
370 370
371 371 @catch_corrupt_db
372 372 def search(self, pattern="*", raw=True, search_raw=True,
373 373 output=False, n=None, unique=False):
374 374 """Search the database using unix glob-style matching (wildcards
375 375 * and ?).
376 376
377 377 Parameters
378 378 ----------
379 379 pattern : str
380 380 The wildcarded pattern to match when searching
381 381 search_raw : bool
382 382 If True, search the raw input, otherwise, the parsed input
383 383 raw, output : bool
384 384 See :meth:`get_range`
385 385 n : None or int
386 386 If an integer is given, it defines the limit of
387 387 returned entries.
388 388 unique : bool
389 389 When it is true, return only unique entries.
390 390
391 391 Returns
392 392 -------
393 393 Tuples as :meth:`get_range`
394 394 """
395 395 tosearch = "source_raw" if search_raw else "source"
396 396 if output:
397 397 tosearch = "history." + tosearch
398 398 self.writeout_cache()
399 399 sqlform = "WHERE %s GLOB ?" % tosearch
400 400 params = (pattern,)
401 401 if unique:
402 402 sqlform += ' GROUP BY {0}'.format(tosearch)
403 403 if n is not None:
404 404 sqlform += " ORDER BY session DESC, line DESC LIMIT ?"
405 405 params += (n,)
406 406 elif unique:
407 407 sqlform += " ORDER BY session, line"
408 408 cur = self._run_sql(sqlform, params, raw=raw, output=output, latest=unique)
409 409 if n is not None:
410 410 return reversed(list(cur))
411 411 return cur
412 412
413 413 @catch_corrupt_db
414 414 def get_range(self, session, start=1, stop=None, raw=True,output=False):
415 415 """Retrieve input by session.
416 416
417 417 Parameters
418 418 ----------
419 419 session : int
420 420 Session number to retrieve.
421 421 start : int
422 422 First line to retrieve.
423 423 stop : int
424 424 End of line range (excluded from output itself). If None, retrieve
425 425 to the end of the session.
426 426 raw : bool
427 427 If True, return untranslated input
428 428 output : bool
429 429 If True, attempt to include output. This will be 'real' Python
430 430 objects for the current session, or text reprs from previous
431 431 sessions if db_log_output was enabled at the time. Where no output
432 432 is found, None is used.
433 433
434 434 Returns
435 435 -------
436 436 entries
437 437 An iterator over the desired lines. Each line is a 3-tuple, either
438 438 (session, line, input) if output is False, or
439 439 (session, line, (input, output)) if output is True.
440 440 """
441 441 if stop:
442 442 lineclause = "line >= ? AND line < ?"
443 443 params = (session, start, stop)
444 444 else:
445 445 lineclause = "line>=?"
446 446 params = (session, start)
447 447
448 448 return self._run_sql("WHERE session==? AND %s" % lineclause,
449 449 params, raw=raw, output=output)
450 450
451 451 def get_range_by_str(self, rangestr, raw=True, output=False):
452 452 """Get lines of history from a string of ranges, as used by magic
453 453 commands %hist, %save, %macro, etc.
454 454
455 455 Parameters
456 456 ----------
457 457 rangestr : str
458 458 A string specifying ranges, e.g. "5 ~2/1-4". If empty string is used,
459 459 this will return everything from current session's history.
460 460
461 461 See the documentation of :func:`%history` for the full details.
462 462
463 463 raw, output : bool
464 464 As :meth:`get_range`
465 465
466 466 Returns
467 467 -------
468 468 Tuples as :meth:`get_range`
469 469 """
470 470 for sess, s, e in extract_hist_ranges(rangestr):
471 471 for line in self.get_range(sess, s, e, raw=raw, output=output):
472 472 yield line
473 473
474 474
475 475 class HistoryManager(HistoryAccessor):
476 476 """A class to organize all history-related functionality in one place.
477 477 """
478 478 # Public interface
479 479
480 480 # An instance of the IPython shell we are attached to
481 481 shell = Instance('IPython.core.interactiveshell.InteractiveShellABC',
482 482 allow_none=True)
483 483 # Lists to hold processed and raw history. These start with a blank entry
484 484 # so that we can index them starting from 1
485 485 input_hist_parsed = List([""])
486 486 input_hist_raw = List([""])
487 487 # A list of directories visited during session
488 488 dir_hist = List()
489 489 @default('dir_hist')
490 490 def _dir_hist_default(self):
491 491 try:
492 492 return [Path.cwd()]
493 493 except OSError:
494 494 return []
495 495
496 496 # A dict of output history, keyed with ints from the shell's
497 497 # execution count.
498 498 output_hist = Dict()
499 499 # The text/plain repr of outputs.
500 500 output_hist_reprs = Dict()
501 501
502 502 # The number of the current session in the history database
503 503 session_number = Integer()
504 504
505 505 db_log_output = Bool(False,
506 506 help="Should the history database include output? (default: no)"
507 507 ).tag(config=True)
508 508 db_cache_size = Integer(0,
509 509 help="Write to database every x commands (higher values save disk access & power).\n"
510 510 "Values of 1 or less effectively disable caching."
511 511 ).tag(config=True)
512 512 # The input and output caches
513 513 db_input_cache = List()
514 514 db_output_cache = List()
515 515
516 516 # History saving in separate thread
517 517 save_thread = Instance('IPython.core.history.HistorySavingThread',
518 518 allow_none=True)
519 519 save_flag = Instance(threading.Event, allow_none=True)
520 520
521 521 # Private interface
522 522 # Variables used to store the three last inputs from the user. On each new
523 523 # history update, we populate the user's namespace with these, shifted as
524 524 # necessary.
525 525 _i00 = Unicode(u'')
526 526 _i = Unicode(u'')
527 527 _ii = Unicode(u'')
528 528 _iii = Unicode(u'')
529 529
530 530 # A regex matching all forms of the exit command, so that we don't store
531 531 # them in the history (it's annoying to rewind the first entry and land on
532 532 # an exit call).
533 533 _exit_re = re.compile(r"(exit|quit)(\s*\(.*\))?$")
534 534
535 535 def __init__(self, shell=None, config=None, **traits):
536 536 """Create a new history manager associated with a shell instance.
537 537 """
538 538 # We need a pointer back to the shell for various tasks.
539 539 super(HistoryManager, self).__init__(shell=shell, config=config,
540 540 **traits)
541 541 self.save_flag = threading.Event()
542 542 self.db_input_cache_lock = threading.Lock()
543 543 self.db_output_cache_lock = threading.Lock()
544 544
545 545 try:
546 546 self.new_session()
547 547 except sqlite3.OperationalError:
548 548 self.log.error("Failed to create history session in %s. History will not be saved.",
549 549 self.hist_file, exc_info=True)
550 550 self.hist_file = ':memory:'
551 551
552 552 if self.enabled and self.hist_file != ':memory:':
553 553 self.save_thread = HistorySavingThread(self)
554 554 self.save_thread.start()
555 555
556 556 def _get_hist_file_name(self, profile=None):
557 557 """Get default history file name based on the Shell's profile.
558 558
559 559 The profile parameter is ignored, but must exist for compatibility with
560 560 the parent class."""
561 561 profile_dir = self.shell.profile_dir.location
562 562 return Path(profile_dir) / "history.sqlite"
563 563
564 564 @only_when_enabled
565 565 def new_session(self, conn=None):
566 566 """Get a new session number."""
567 567 if conn is None:
568 568 conn = self.db
569 569
570 570 with conn:
571 571 cur = conn.execute("""INSERT INTO sessions VALUES (NULL, ?, NULL,
572 572 NULL, "") """, (datetime.datetime.now(),))
573 573 self.session_number = cur.lastrowid
574 574
575 575 def end_session(self):
576 576 """Close the database session, filling in the end time and line count."""
577 577 self.writeout_cache()
578 578 with self.db:
579 579 self.db.execute("""UPDATE sessions SET end=?, num_cmds=? WHERE
580 580 session==?""", (datetime.datetime.now(),
581 581 len(self.input_hist_parsed)-1, self.session_number))
582 582 self.session_number = 0
583 583
584 584 def name_session(self, name):
585 585 """Give the current session a name in the history database."""
586 586 with self.db:
587 587 self.db.execute("UPDATE sessions SET remark=? WHERE session==?",
588 588 (name, self.session_number))
589 589
590 590 def reset(self, new_session=True):
591 591 """Clear the session history, releasing all object references, and
592 592 optionally open a new session."""
593 593 self.output_hist.clear()
594 594 # The directory history can't be completely empty
595 595 self.dir_hist[:] = [Path.cwd()]
596 596
597 597 if new_session:
598 598 if self.session_number:
599 599 self.end_session()
600 600 self.input_hist_parsed[:] = [""]
601 601 self.input_hist_raw[:] = [""]
602 602 self.new_session()
603 603
604 604 # ------------------------------
605 605 # Methods for retrieving history
606 606 # ------------------------------
607 607 def get_session_info(self, session=0):
608 608 """Get info about a session.
609 609
610 610 Parameters
611 611 ----------
612 612 session : int
613 613 Session number to retrieve. The current session is 0, and negative
614 614 numbers count back from current session, so -1 is the previous session.
615 615
616 616 Returns
617 617 -------
618 618 session_id : int
619 619 Session ID number
620 620 start : datetime
621 621 Timestamp for the start of the session.
622 622 end : datetime
623 623 Timestamp for the end of the session, or None if IPython crashed.
624 624 num_cmds : int
625 625 Number of commands run, or None if IPython crashed.
626 626 remark : unicode
627 627 A manually set description.
628 628 """
629 629 if session <= 0:
630 630 session += self.session_number
631 631
632 632 return super(HistoryManager, self).get_session_info(session=session)
633 633
634 634 def _get_range_session(self, start=1, stop=None, raw=True, output=False):
635 635 """Get input and output history from the current session. Called by
636 636 get_range, and takes similar parameters."""
637 637 input_hist = self.input_hist_raw if raw else self.input_hist_parsed
638 638
639 639 n = len(input_hist)
640 640 if start < 0:
641 641 start += n
642 642 if not stop or (stop > n):
643 643 stop = n
644 644 elif stop < 0:
645 645 stop += n
646 646
647 647 for i in range(start, stop):
648 648 if output:
649 649 line = (input_hist[i], self.output_hist_reprs.get(i))
650 650 else:
651 651 line = input_hist[i]
652 652 yield (0, i, line)
653 653
654 654 def get_range(self, session=0, start=1, stop=None, raw=True,output=False):
655 655 """Retrieve input by session.
656 656
657 657 Parameters
658 658 ----------
659 659 session : int
660 660 Session number to retrieve. The current session is 0, and negative
661 661 numbers count back from current session, so -1 is previous session.
662 662 start : int
663 663 First line to retrieve.
664 664 stop : int
665 665 End of line range (excluded from output itself). If None, retrieve
666 666 to the end of the session.
667 667 raw : bool
668 668 If True, return untranslated input
669 669 output : bool
670 670 If True, attempt to include output. This will be 'real' Python
671 671 objects for the current session, or text reprs from previous
672 672 sessions if db_log_output was enabled at the time. Where no output
673 673 is found, None is used.
674 674
675 675 Returns
676 676 -------
677 677 entries
678 678 An iterator over the desired lines. Each line is a 3-tuple, either
679 679 (session, line, input) if output is False, or
680 680 (session, line, (input, output)) if output is True.
681 681 """
682 682 if session <= 0:
683 683 session += self.session_number
684 684 if session==self.session_number: # Current session
685 685 return self._get_range_session(start, stop, raw, output)
686 686 return super(HistoryManager, self).get_range(session, start, stop, raw,
687 687 output)
688 688
689 689 ## ----------------------------
690 690 ## Methods for storing history:
691 691 ## ----------------------------
692 692 def store_inputs(self, line_num, source, source_raw=None):
693 693 """Store source and raw input in history and create input cache
694 694 variables ``_i*``.
695 695
696 696 Parameters
697 697 ----------
698 698 line_num : int
699 699 The prompt number of this input.
700 700 source : str
701 701 Python input.
702 702 source_raw : str, optional
703 703 If given, this is the raw input without any IPython transformations
704 704 applied to it. If not given, ``source`` is used.
705 705 """
706 706 if source_raw is None:
707 707 source_raw = source
708 708 source = source.rstrip('\n')
709 709 source_raw = source_raw.rstrip('\n')
710 710
711 711 # do not store exit/quit commands
712 712 if self._exit_re.match(source_raw.strip()):
713 713 return
714 714
715 715 self.input_hist_parsed.append(source)
716 716 self.input_hist_raw.append(source_raw)
717 717
718 718 with self.db_input_cache_lock:
719 719 self.db_input_cache.append((line_num, source, source_raw))
720 720 # Trigger to flush cache and write to DB.
721 721 if len(self.db_input_cache) >= self.db_cache_size:
722 722 self.save_flag.set()
723 723
724 724 # update the auto _i variables
725 725 self._iii = self._ii
726 726 self._ii = self._i
727 727 self._i = self._i00
728 728 self._i00 = source_raw
729 729
730 730 # hackish access to user namespace to create _i1,_i2... dynamically
731 731 new_i = '_i%s' % line_num
732 732 to_main = {'_i': self._i,
733 733 '_ii': self._ii,
734 734 '_iii': self._iii,
735 735 new_i : self._i00 }
736 736
737 737 if self.shell is not None:
738 738 self.shell.push(to_main, interactive=False)
739 739
740 740 def store_output(self, line_num):
741 741 """If database output logging is enabled, this saves all the
742 742 outputs from the indicated prompt number to the database. It's
743 743 called by run_cell after code has been executed.
744 744
745 745 Parameters
746 746 ----------
747 747 line_num : int
748 748 The line number from which to save outputs
749 749 """
750 750 if (not self.db_log_output) or (line_num not in self.output_hist_reprs):
751 751 return
752 752 output = self.output_hist_reprs[line_num]
753 753
754 754 with self.db_output_cache_lock:
755 755 self.db_output_cache.append((line_num, output))
756 756 if self.db_cache_size <= 1:
757 757 self.save_flag.set()
758 758
759 759 def _writeout_input_cache(self, conn):
760 760 with conn:
761 761 for line in self.db_input_cache:
762 762 conn.execute("INSERT INTO history VALUES (?, ?, ?, ?)",
763 763 (self.session_number,)+line)
764 764
765 765 def _writeout_output_cache(self, conn):
766 766 with conn:
767 767 for line in self.db_output_cache:
768 768 conn.execute("INSERT INTO output_history VALUES (?, ?, ?)",
769 769 (self.session_number,)+line)
770 770
771 771 @only_when_enabled
772 772 def writeout_cache(self, conn=None):
773 773 """Write any entries in the cache to the database."""
774 774 if conn is None:
775 775 conn = self.db
776 776
777 777 with self.db_input_cache_lock:
778 778 try:
779 779 self._writeout_input_cache(conn)
780 780 except sqlite3.IntegrityError:
781 781 self.new_session(conn)
782 782 print("ERROR! Session/line number was not unique in",
783 783 "database. History logging moved to new session",
784 784 self.session_number)
785 785 try:
786 786 # Try writing to the new session. If this fails, don't
787 787 # recurse
788 788 self._writeout_input_cache(conn)
789 789 except sqlite3.IntegrityError:
790 790 pass
791 791 finally:
792 792 self.db_input_cache = []
793 793
794 794 with self.db_output_cache_lock:
795 795 try:
796 796 self._writeout_output_cache(conn)
797 797 except sqlite3.IntegrityError:
798 798 print("!! Session/line number for output was not unique",
799 799 "in database. Output will not be stored.")
800 800 finally:
801 801 self.db_output_cache = []
802 802
803 803
804 804 class HistorySavingThread(threading.Thread):
805 805 """This thread takes care of writing history to the database, so that
806 806 the UI isn't held up while that happens.
807 807
808 808 It waits for the HistoryManager's save_flag to be set, then writes out
809 809 the history cache. The main thread is responsible for setting the flag when
810 810 the cache size reaches a defined threshold."""
811 811 daemon = True
812 812 stop_now = False
813 813 enabled = True
814 814 def __init__(self, history_manager):
815 815 super(HistorySavingThread, self).__init__(name="IPythonHistorySavingThread")
816 816 self.history_manager = history_manager
817 817 self.enabled = history_manager.enabled
818 818 atexit.register(self.stop)
819 819
820 820 @only_when_enabled
821 821 def run(self):
822 822 # We need a separate db connection per thread:
823 823 try:
824 824 self.db = sqlite3.connect(
825 825 str(self.history_manager.hist_file),
826 826 **self.history_manager.connection_options,
827 827 )
828 828 while True:
829 829 self.history_manager.save_flag.wait()
830 830 if self.stop_now:
831 831 self.db.close()
832 832 return
833 833 self.history_manager.save_flag.clear()
834 834 self.history_manager.writeout_cache(self.db)
835 835 except Exception as e:
836 836 print(("The history saving thread hit an unexpected error (%s)."
837 837 "History will not be written to the database.") % repr(e))
838 838
839 839 def stop(self):
840 840 """This can be called from the main thread to safely stop this thread.
841 841
842 842 Note that it does not attempt to write out remaining history before
843 843 exiting. That should be done by calling the HistoryManager's
844 844 end_session method."""
845 845 self.stop_now = True
846 846 self.history_manager.save_flag.set()
847 847 self.join()
848 848
849 849
850 850 # To match, e.g. ~5/8-~2/3
851 851 range_re = re.compile(r"""
852 852 ((?P<startsess>~?\d+)/)?
853 853 (?P<start>\d+)?
854 854 ((?P<sep>[\-:])
855 855 ((?P<endsess>~?\d+)/)?
856 856 (?P<end>\d+))?
857 857 $""", re.VERBOSE)
858 858
859 859
860 860 def extract_hist_ranges(ranges_str):
861 861 """Turn a string of history ranges into 3-tuples of (session, start, stop).
862 862
863 863 Empty string results in a `[(0, 1, None)]`, i.e. "everything from current
864 864 session".
865 865
866 866 Examples
867 867 --------
868 868 >>> list(extract_hist_ranges("~8/5-~7/4 2"))
869 869 [(-8, 5, None), (-7, 1, 5), (0, 2, 3)]
870 870 """
871 871 if ranges_str == "":
872 872 yield (0, 1, None) # Everything from current session
873 873 return
874 874
875 875 for range_str in ranges_str.split():
876 876 rmatch = range_re.match(range_str)
877 877 if not rmatch:
878 878 continue
879 879 start = rmatch.group("start")
880 880 if start:
881 881 start = int(start)
882 882 end = rmatch.group("end")
883 883 # If no end specified, get (a, a + 1)
884 884 end = int(end) if end else start + 1
885 885 else: # start not specified
886 886 if not rmatch.group('startsess'): # no startsess
887 887 continue
888 888 start = 1
889 889 end = None # provide the entire session hist
890 890
891 891 if rmatch.group("sep") == "-": # 1-3 == 1:4 --> [1, 2, 3]
892 892 end += 1
893 893 startsess = rmatch.group("startsess") or "0"
894 894 endsess = rmatch.group("endsess") or startsess
895 895 startsess = int(startsess.replace("~","-"))
896 896 endsess = int(endsess.replace("~","-"))
897 897 assert endsess >= startsess, "start session must be earlier than end session"
898 898
899 899 if endsess == startsess:
900 900 yield (startsess, start, end)
901 901 continue
902 902 # Multiple sessions in one range:
903 903 yield (startsess, start, None)
904 904 for sess in range(startsess+1, endsess):
905 905 yield (sess, 1, None)
906 906 yield (endsess, 1, end)
907 907
908 908
909 909 def _format_lineno(session, line):
910 910 """Helper function to format line numbers properly."""
911 911 if session == 0:
912 912 return str(line)
913 913 return "%s#%s" % (session, line)
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
@@ -1,1510 +1,1510 b''
1 1 # -*- coding: utf-8 -*-
2 2 """Implementation of execution-related magic functions."""
3 3
4 4 # Copyright (c) IPython Development Team.
5 5 # Distributed under the terms of the Modified BSD License.
6 6
7 7
8 8 import ast
9 9 import bdb
10 10 import builtins as builtin_mod
11 11 import cProfile as profile
12 12 import gc
13 13 import itertools
14 14 import math
15 15 import os
16 16 import pstats
17 17 import re
18 18 import shlex
19 19 import sys
20 20 import time
21 21 import timeit
22 22 from ast import Module
23 23 from io import StringIO
24 24 from logging import error
25 25 from pathlib import Path
26 26 from pdb import Restart
27 27 from warnings import warn
28 28
29 29 from IPython.core import magic_arguments, oinspect, page
30 30 from IPython.core.error import UsageError
31 31 from IPython.core.macro import Macro
32 32 from IPython.core.magic import (
33 33 Magics,
34 34 cell_magic,
35 35 line_cell_magic,
36 36 line_magic,
37 37 magics_class,
38 38 needs_local_scope,
39 39 no_var_expand,
40 40 on_off,
41 41 )
42 42 from IPython.testing.skipdoctest import skip_doctest
43 43 from IPython.utils.capture import capture_output
44 44 from IPython.utils.contexts import preserve_keys
45 45 from IPython.utils.ipstruct import Struct
46 46 from IPython.utils.module_paths import find_mod
47 47 from IPython.utils.path import get_py_filename, shellglob
48 48 from IPython.utils.timing import clock, clock2
49 49
50 50 #-----------------------------------------------------------------------------
51 51 # Magic implementation classes
52 52 #-----------------------------------------------------------------------------
53 53
54 54
55 55 class TimeitResult(object):
56 56 """
57 57 Object returned by the timeit magic with info about the run.
58 58
59 59 Contains the following attributes :
60 60
61 61 loops: (int) number of loops done per measurement
62 62 repeat: (int) number of times the measurement has been repeated
63 63 best: (float) best execution time / number
64 64 all_runs: (list of float) execution time of each run (in s)
65 65 compile_time: (float) time of statement compilation (s)
66 66
67 67 """
68 68 def __init__(self, loops, repeat, best, worst, all_runs, compile_time, precision):
69 69 self.loops = loops
70 70 self.repeat = repeat
71 71 self.best = best
72 72 self.worst = worst
73 73 self.all_runs = all_runs
74 74 self.compile_time = compile_time
75 75 self._precision = precision
76 76 self.timings = [ dt / self.loops for dt in all_runs]
77 77
78 78 @property
79 79 def average(self):
80 80 return math.fsum(self.timings) / len(self.timings)
81 81
82 82 @property
83 83 def stdev(self):
84 84 mean = self.average
85 85 return (math.fsum([(x - mean) ** 2 for x in self.timings]) / len(self.timings)) ** 0.5
86 86
87 87 def __str__(self):
88 88 pm = '+-'
89 89 if hasattr(sys.stdout, 'encoding') and sys.stdout.encoding:
90 90 try:
91 91 u'\xb1'.encode(sys.stdout.encoding)
92 92 pm = u'\xb1'
93 93 except:
94 94 pass
95 95 return "{mean} {pm} {std} per loop (mean {pm} std. dev. of {runs} run{run_plural}, {loops:,} loop{loop_plural} each)".format(
96 96 pm=pm,
97 97 runs=self.repeat,
98 98 loops=self.loops,
99 99 loop_plural="" if self.loops == 1 else "s",
100 100 run_plural="" if self.repeat == 1 else "s",
101 101 mean=_format_time(self.average, self._precision),
102 102 std=_format_time(self.stdev, self._precision),
103 103 )
104 104
105 105 def _repr_pretty_(self, p , cycle):
106 106 unic = self.__str__()
107 107 p.text(u'<TimeitResult : '+unic+u'>')
108 108
109 109
110 110 class TimeitTemplateFiller(ast.NodeTransformer):
111 111 """Fill in the AST template for timing execution.
112 112
113 113 This is quite closely tied to the template definition, which is in
114 114 :meth:`ExecutionMagics.timeit`.
115 115 """
116 116 def __init__(self, ast_setup, ast_stmt):
117 117 self.ast_setup = ast_setup
118 118 self.ast_stmt = ast_stmt
119 119
120 120 def visit_FunctionDef(self, node):
121 121 "Fill in the setup statement"
122 122 self.generic_visit(node)
123 123 if node.name == "inner":
124 124 node.body[:1] = self.ast_setup.body
125 125
126 126 return node
127 127
128 128 def visit_For(self, node):
129 129 "Fill in the statement to be timed"
130 130 if getattr(getattr(node.body[0], 'value', None), 'id', None) == 'stmt':
131 131 node.body = self.ast_stmt.body
132 132 return node
133 133
134 134
135 135 class Timer(timeit.Timer):
136 136 """Timer class that explicitly uses self.inner
137 137
138 138 which is an undocumented implementation detail of CPython,
139 139 not shared by PyPy.
140 140 """
141 141 # Timer.timeit copied from CPython 3.4.2
142 142 def timeit(self, number=timeit.default_number):
143 143 """Time 'number' executions of the main statement.
144 144
145 145 To be precise, this executes the setup statement once, and
146 146 then returns the time it takes to execute the main statement
147 147 a number of times, as a float measured in seconds. The
148 148 argument is the number of times through the loop, defaulting
149 149 to one million. The main statement, the setup statement and
150 150 the timer function to be used are passed to the constructor.
151 151 """
152 152 it = itertools.repeat(None, number)
153 153 gcold = gc.isenabled()
154 154 gc.disable()
155 155 try:
156 156 timing = self.inner(it, self.timer)
157 157 finally:
158 158 if gcold:
159 159 gc.enable()
160 160 return timing
161 161
162 162
163 163 @magics_class
164 164 class ExecutionMagics(Magics):
165 165 """Magics related to code execution, debugging, profiling, etc.
166 166
167 167 """
168 168
169 169 def __init__(self, shell):
170 170 super(ExecutionMagics, self).__init__(shell)
171 171 # Default execution function used to actually run user code.
172 172 self.default_runner = None
173 173
174 174 @skip_doctest
175 175 @no_var_expand
176 176 @line_cell_magic
177 177 def prun(self, parameter_s='', cell=None):
178 178
179 179 """Run a statement through the python code profiler.
180 180
181 181 Usage, in line mode:
182 182 %prun [options] statement
183 183
184 184 Usage, in cell mode:
185 185 %%prun [options] [statement]
186 186 code...
187 187 code...
188 188
189 189 In cell mode, the additional code lines are appended to the (possibly
190 190 empty) statement in the first line. Cell mode allows you to easily
191 191 profile multiline blocks without having to put them in a separate
192 192 function.
193 193
194 194 The given statement (which doesn't require quote marks) is run via the
195 195 python profiler in a manner similar to the profile.run() function.
196 196 Namespaces are internally managed to work correctly; profile.run
197 197 cannot be used in IPython because it makes certain assumptions about
198 198 namespaces which do not hold under IPython.
199 199
200 200 Options:
201 201
202 202 -l <limit>
203 203 you can place restrictions on what or how much of the
204 204 profile gets printed. The limit value can be:
205 205
206 206 * A string: only information for function names containing this string
207 207 is printed.
208 208
209 209 * An integer: only these many lines are printed.
210 210
211 211 * A float (between 0 and 1): this fraction of the report is printed
212 212 (for example, use a limit of 0.4 to see the topmost 40% only).
213 213
214 214 You can combine several limits with repeated use of the option. For
215 215 example, ``-l __init__ -l 5`` will print only the topmost 5 lines of
216 216 information about class constructors.
217 217
218 218 -r
219 219 return the pstats.Stats object generated by the profiling. This
220 220 object has all the information about the profile in it, and you can
221 221 later use it for further analysis or in other functions.
222 222
223 223 -s <key>
224 224 sort profile by given key. You can provide more than one key
225 225 by using the option several times: '-s key1 -s key2 -s key3...'. The
226 226 default sorting key is 'time'.
227 227
228 228 The following is copied verbatim from the profile documentation
229 229 referenced below:
230 230
231 231 When more than one key is provided, additional keys are used as
232 232 secondary criteria when the there is equality in all keys selected
233 233 before them.
234 234
235 235 Abbreviations can be used for any key names, as long as the
236 236 abbreviation is unambiguous. The following are the keys currently
237 237 defined:
238 238
239 239 ============ =====================
240 240 Valid Arg Meaning
241 241 ============ =====================
242 242 "calls" call count
243 243 "cumulative" cumulative time
244 244 "file" file name
245 245 "module" file name
246 246 "pcalls" primitive call count
247 247 "line" line number
248 248 "name" function name
249 249 "nfl" name/file/line
250 250 "stdname" standard name
251 251 "time" internal time
252 252 ============ =====================
253 253
254 254 Note that all sorts on statistics are in descending order (placing
255 255 most time consuming items first), where as name, file, and line number
256 256 searches are in ascending order (i.e., alphabetical). The subtle
257 257 distinction between "nfl" and "stdname" is that the standard name is a
258 258 sort of the name as printed, which means that the embedded line
259 259 numbers get compared in an odd way. For example, lines 3, 20, and 40
260 260 would (if the file names were the same) appear in the string order
261 261 "20" "3" and "40". In contrast, "nfl" does a numeric compare of the
262 262 line numbers. In fact, sort_stats("nfl") is the same as
263 263 sort_stats("name", "file", "line").
264 264
265 265 -T <filename>
266 266 save profile results as shown on screen to a text
267 267 file. The profile is still shown on screen.
268 268
269 269 -D <filename>
270 270 save (via dump_stats) profile statistics to given
271 271 filename. This data is in a format understood by the pstats module, and
272 272 is generated by a call to the dump_stats() method of profile
273 273 objects. The profile is still shown on screen.
274 274
275 275 -q
276 276 suppress output to the pager. Best used with -T and/or -D above.
277 277
278 278 If you want to run complete programs under the profiler's control, use
279 279 ``%run -p [prof_opts] filename.py [args to program]`` where prof_opts
280 280 contains profiler specific options as described here.
281 281
282 282 You can read the complete documentation for the profile module with::
283 283
284 284 In [1]: import profile; profile.help()
285 285
286 286 .. versionchanged:: 7.3
287 287 User variables are no longer expanded,
288 288 the magic line is always left unmodified.
289 289
290 290 """
291 291 opts, arg_str = self.parse_options(parameter_s, 'D:l:rs:T:q',
292 292 list_all=True, posix=False)
293 293 if cell is not None:
294 294 arg_str += '\n' + cell
295 295 arg_str = self.shell.transform_cell(arg_str)
296 296 return self._run_with_profiler(arg_str, opts, self.shell.user_ns)
297 297
298 298 def _run_with_profiler(self, code, opts, namespace):
299 299 """
300 300 Run `code` with profiler. Used by ``%prun`` and ``%run -p``.
301 301
302 302 Parameters
303 303 ----------
304 304 code : str
305 305 Code to be executed.
306 306 opts : Struct
307 307 Options parsed by `self.parse_options`.
308 308 namespace : dict
309 309 A dictionary for Python namespace (e.g., `self.shell.user_ns`).
310 310
311 311 """
312 312
313 313 # Fill default values for unspecified options:
314 314 opts.merge(Struct(D=[''], l=[], s=['time'], T=['']))
315 315
316 316 prof = profile.Profile()
317 317 try:
318 318 prof = prof.runctx(code, namespace, namespace)
319 319 sys_exit = ''
320 320 except SystemExit:
321 321 sys_exit = """*** SystemExit exception caught in code being profiled."""
322 322
323 323 stats = pstats.Stats(prof).strip_dirs().sort_stats(*opts.s)
324 324
325 325 lims = opts.l
326 326 if lims:
327 327 lims = [] # rebuild lims with ints/floats/strings
328 328 for lim in opts.l:
329 329 try:
330 330 lims.append(int(lim))
331 331 except ValueError:
332 332 try:
333 333 lims.append(float(lim))
334 334 except ValueError:
335 335 lims.append(lim)
336 336
337 337 # Trap output.
338 338 stdout_trap = StringIO()
339 339 stats_stream = stats.stream
340 340 try:
341 341 stats.stream = stdout_trap
342 342 stats.print_stats(*lims)
343 343 finally:
344 344 stats.stream = stats_stream
345 345
346 346 output = stdout_trap.getvalue()
347 347 output = output.rstrip()
348 348
349 349 if 'q' not in opts:
350 350 page.page(output)
351 351 print(sys_exit, end=' ')
352 352
353 353 dump_file = opts.D[0]
354 354 text_file = opts.T[0]
355 355 if dump_file:
356 356 prof.dump_stats(dump_file)
357 357 print(
358 358 f"\n*** Profile stats marshalled to file {repr(dump_file)}.{sys_exit}"
359 359 )
360 360 if text_file:
361 361 pfile = Path(text_file)
362 362 pfile.touch(exist_ok=True)
363 363 pfile.write_text(output, encoding="utf-8")
364 364
365 365 print(
366 366 f"\n*** Profile printout saved to text file {repr(text_file)}.{sys_exit}"
367 367 )
368 368
369 369 if 'r' in opts:
370 370 return stats
371 371
372 372 return None
373 373
374 374 @line_magic
375 375 def pdb(self, parameter_s=''):
376 376 """Control the automatic calling of the pdb interactive debugger.
377 377
378 378 Call as '%pdb on', '%pdb 1', '%pdb off' or '%pdb 0'. If called without
379 379 argument it works as a toggle.
380 380
381 381 When an exception is triggered, IPython can optionally call the
382 382 interactive pdb debugger after the traceback printout. %pdb toggles
383 383 this feature on and off.
384 384
385 385 The initial state of this feature is set in your configuration
386 386 file (the option is ``InteractiveShell.pdb``).
387 387
388 388 If you want to just activate the debugger AFTER an exception has fired,
389 389 without having to type '%pdb on' and rerunning your code, you can use
390 390 the %debug magic."""
391 391
392 392 par = parameter_s.strip().lower()
393 393
394 394 if par:
395 395 try:
396 396 new_pdb = {'off':0,'0':0,'on':1,'1':1}[par]
397 397 except KeyError:
398 398 print ('Incorrect argument. Use on/1, off/0, '
399 399 'or nothing for a toggle.')
400 400 return
401 401 else:
402 402 # toggle
403 403 new_pdb = not self.shell.call_pdb
404 404
405 405 # set on the shell
406 406 self.shell.call_pdb = new_pdb
407 407 print('Automatic pdb calling has been turned',on_off(new_pdb))
408 408
409 409 @magic_arguments.magic_arguments()
410 410 @magic_arguments.argument('--breakpoint', '-b', metavar='FILE:LINE',
411 411 help="""
412 412 Set break point at LINE in FILE.
413 413 """
414 414 )
415 415 @magic_arguments.argument('statement', nargs='*',
416 416 help="""
417 417 Code to run in debugger.
418 418 You can omit this in cell magic mode.
419 419 """
420 420 )
421 421 @no_var_expand
422 422 @line_cell_magic
423 423 def debug(self, line='', cell=None):
424 424 """Activate the interactive debugger.
425 425
426 426 This magic command support two ways of activating debugger.
427 427 One is to activate debugger before executing code. This way, you
428 428 can set a break point, to step through the code from the point.
429 429 You can use this mode by giving statements to execute and optionally
430 430 a breakpoint.
431 431
432 432 The other one is to activate debugger in post-mortem mode. You can
433 433 activate this mode simply running %debug without any argument.
434 434 If an exception has just occurred, this lets you inspect its stack
435 435 frames interactively. Note that this will always work only on the last
436 436 traceback that occurred, so you must call this quickly after an
437 437 exception that you wish to inspect has fired, because if another one
438 438 occurs, it clobbers the previous one.
439 439
440 440 If you want IPython to automatically do this on every exception, see
441 441 the %pdb magic for more details.
442 442
443 443 .. versionchanged:: 7.3
444 444 When running code, user variables are no longer expanded,
445 445 the magic line is always left unmodified.
446 446
447 447 """
448 448 args = magic_arguments.parse_argstring(self.debug, line)
449 449
450 450 if not (args.breakpoint or args.statement or cell):
451 451 self._debug_post_mortem()
452 452 elif not (args.breakpoint or cell):
453 453 # If there is no breakpoints, the line is just code to execute
454 454 self._debug_exec(line, None)
455 455 else:
456 456 # Here we try to reconstruct the code from the output of
457 457 # parse_argstring. This might not work if the code has spaces
458 458 # For example this fails for `print("a b")`
459 459 code = "\n".join(args.statement)
460 460 if cell:
461 461 code += "\n" + cell
462 462 self._debug_exec(code, args.breakpoint)
463 463
464 464 def _debug_post_mortem(self):
465 465 self.shell.debugger(force=True)
466 466
467 467 def _debug_exec(self, code, breakpoint):
468 468 if breakpoint:
469 469 (filename, bp_line) = breakpoint.rsplit(':', 1)
470 470 bp_line = int(bp_line)
471 471 else:
472 472 (filename, bp_line) = (None, None)
473 473 self._run_with_debugger(code, self.shell.user_ns, filename, bp_line)
474 474
475 475 @line_magic
476 476 def tb(self, s):
477 477 """Print the last traceback.
478 478
479 479 Optionally, specify an exception reporting mode, tuning the
480 480 verbosity of the traceback. By default the currently-active exception
481 481 mode is used. See %xmode for changing exception reporting modes.
482 482
483 483 Valid modes: Plain, Context, Verbose, and Minimal.
484 484 """
485 485 interactive_tb = self.shell.InteractiveTB
486 486 if s:
487 487 # Switch exception reporting mode for this one call.
488 488 # Ensure it is switched back.
489 489 def xmode_switch_err(name):
490 490 warn('Error changing %s exception modes.\n%s' %
491 491 (name,sys.exc_info()[1]))
492 492
493 493 new_mode = s.strip().capitalize()
494 494 original_mode = interactive_tb.mode
495 495 try:
496 496 try:
497 497 interactive_tb.set_mode(mode=new_mode)
498 498 except Exception:
499 499 xmode_switch_err('user')
500 500 else:
501 501 self.shell.showtraceback()
502 502 finally:
503 503 interactive_tb.set_mode(mode=original_mode)
504 504 else:
505 505 self.shell.showtraceback()
506 506
507 507 @skip_doctest
508 508 @line_magic
509 509 def run(self, parameter_s='', runner=None,
510 510 file_finder=get_py_filename):
511 511 """Run the named file inside IPython as a program.
512 512
513 513 Usage::
514
514
515 515 %run [-n -i -e -G]
516 516 [( -t [-N<N>] | -d [-b<N>] | -p [profile options] )]
517 517 ( -m mod | filename ) [args]
518 518
519 519 The filename argument should be either a pure Python script (with
520 520 extension ``.py``), or a file with custom IPython syntax (such as
521 521 magics). If the latter, the file can be either a script with ``.ipy``
522 522 extension, or a Jupyter notebook with ``.ipynb`` extension. When running
523 523 a Jupyter notebook, the output from print statements and other
524 524 displayed objects will appear in the terminal (even matplotlib figures
525 525 will open, if a terminal-compliant backend is being used). Note that,
526 526 at the system command line, the ``jupyter run`` command offers similar
527 527 functionality for executing notebooks (albeit currently with some
528 528 differences in supported options).
529 529
530 530 Parameters after the filename are passed as command-line arguments to
531 531 the program (put in sys.argv). Then, control returns to IPython's
532 532 prompt.
533 533
534 534 This is similar to running at a system prompt ``python file args``,
535 535 but with the advantage of giving you IPython's tracebacks, and of
536 536 loading all variables into your interactive namespace for further use
537 537 (unless -p is used, see below).
538 538
539 539 The file is executed in a namespace initially consisting only of
540 540 ``__name__=='__main__'`` and sys.argv constructed as indicated. It thus
541 541 sees its environment as if it were being run as a stand-alone program
542 542 (except for sharing global objects such as previously imported
543 543 modules). But after execution, the IPython interactive namespace gets
544 544 updated with all variables defined in the program (except for __name__
545 545 and sys.argv). This allows for very convenient loading of code for
546 546 interactive work, while giving each program a 'clean sheet' to run in.
547 547
548 548 Arguments are expanded using shell-like glob match. Patterns
549 549 '*', '?', '[seq]' and '[!seq]' can be used. Additionally,
550 550 tilde '~' will be expanded into user's home directory. Unlike
551 551 real shells, quotation does not suppress expansions. Use
552 552 *two* back slashes (e.g. ``\\\\*``) to suppress expansions.
553 553 To completely disable these expansions, you can use -G flag.
554 554
555 On Windows systems, the use of single quotes `'` when specifying
555 On Windows systems, the use of single quotes `'` when specifying
556 556 a file is not supported. Use double quotes `"`.
557 557
558 558 Options:
559 559
560 560 -n
561 561 __name__ is NOT set to '__main__', but to the running file's name
562 562 without extension (as python does under import). This allows running
563 563 scripts and reloading the definitions in them without calling code
564 564 protected by an ``if __name__ == "__main__"`` clause.
565 565
566 566 -i
567 567 run the file in IPython's namespace instead of an empty one. This
568 568 is useful if you are experimenting with code written in a text editor
569 569 which depends on variables defined interactively.
570 570
571 571 -e
572 572 ignore sys.exit() calls or SystemExit exceptions in the script
573 573 being run. This is particularly useful if IPython is being used to
574 574 run unittests, which always exit with a sys.exit() call. In such
575 575 cases you are interested in the output of the test results, not in
576 576 seeing a traceback of the unittest module.
577 577
578 578 -t
579 579 print timing information at the end of the run. IPython will give
580 580 you an estimated CPU time consumption for your script, which under
581 581 Unix uses the resource module to avoid the wraparound problems of
582 582 time.clock(). Under Unix, an estimate of time spent on system tasks
583 583 is also given (for Windows platforms this is reported as 0.0).
584 584
585 585 If -t is given, an additional ``-N<N>`` option can be given, where <N>
586 586 must be an integer indicating how many times you want the script to
587 587 run. The final timing report will include total and per run results.
588 588
589 589 For example (testing the script uniq_stable.py)::
590 590
591 591 In [1]: run -t uniq_stable
592 592
593 593 IPython CPU timings (estimated):
594 594 User : 0.19597 s.
595 595 System: 0.0 s.
596 596
597 597 In [2]: run -t -N5 uniq_stable
598 598
599 599 IPython CPU timings (estimated):
600 600 Total runs performed: 5
601 601 Times : Total Per run
602 602 User : 0.910862 s, 0.1821724 s.
603 603 System: 0.0 s, 0.0 s.
604 604
605 605 -d
606 606 run your program under the control of pdb, the Python debugger.
607 607 This allows you to execute your program step by step, watch variables,
608 608 etc. Internally, what IPython does is similar to calling::
609 609
610 610 pdb.run('execfile("YOURFILENAME")')
611 611
612 612 with a breakpoint set on line 1 of your file. You can change the line
613 613 number for this automatic breakpoint to be <N> by using the -bN option
614 614 (where N must be an integer). For example::
615 615
616 616 %run -d -b40 myscript
617 617
618 618 will set the first breakpoint at line 40 in myscript.py. Note that
619 619 the first breakpoint must be set on a line which actually does
620 620 something (not a comment or docstring) for it to stop execution.
621 621
622 622 Or you can specify a breakpoint in a different file::
623 623
624 624 %run -d -b myotherfile.py:20 myscript
625 625
626 626 When the pdb debugger starts, you will see a (Pdb) prompt. You must
627 627 first enter 'c' (without quotes) to start execution up to the first
628 628 breakpoint.
629 629
630 630 Entering 'help' gives information about the use of the debugger. You
631 631 can easily see pdb's full documentation with "import pdb;pdb.help()"
632 632 at a prompt.
633 633
634 634 -p
635 635 run program under the control of the Python profiler module (which
636 636 prints a detailed report of execution times, function calls, etc).
637 637
638 638 You can pass other options after -p which affect the behavior of the
639 639 profiler itself. See the docs for %prun for details.
640 640
641 641 In this mode, the program's variables do NOT propagate back to the
642 642 IPython interactive namespace (because they remain in the namespace
643 643 where the profiler executes them).
644 644
645 645 Internally this triggers a call to %prun, see its documentation for
646 646 details on the options available specifically for profiling.
647 647
648 648 There is one special usage for which the text above doesn't apply:
649 649 if the filename ends with .ipy[nb], the file is run as ipython script,
650 650 just as if the commands were written on IPython prompt.
651 651
652 652 -m
653 653 specify module name to load instead of script path. Similar to
654 654 the -m option for the python interpreter. Use this option last if you
655 655 want to combine with other %run options. Unlike the python interpreter
656 656 only source modules are allowed no .pyc or .pyo files.
657 657 For example::
658 658
659 659 %run -m example
660 660
661 661 will run the example module.
662 662
663 663 -G
664 664 disable shell-like glob expansion of arguments.
665 665
666 666 """
667 667
668 668 # Logic to handle issue #3664
669 669 # Add '--' after '-m <module_name>' to ignore additional args passed to a module.
670 670 if '-m' in parameter_s and '--' not in parameter_s:
671 671 argv = shlex.split(parameter_s, posix=(os.name == 'posix'))
672 672 for idx, arg in enumerate(argv):
673 673 if arg and arg.startswith('-') and arg != '-':
674 674 if arg == '-m':
675 675 argv.insert(idx + 2, '--')
676 676 break
677 677 else:
678 678 # Positional arg, break
679 679 break
680 680 parameter_s = ' '.join(shlex.quote(arg) for arg in argv)
681 681
682 682 # get arguments and set sys.argv for program to be run.
683 683 opts, arg_lst = self.parse_options(parameter_s,
684 684 'nidtN:b:pD:l:rs:T:em:G',
685 685 mode='list', list_all=1)
686 686 if "m" in opts:
687 687 modulename = opts["m"][0]
688 688 modpath = find_mod(modulename)
689 689 if modpath is None:
690 690 msg = '%r is not a valid modulename on sys.path'%modulename
691 691 raise Exception(msg)
692 692 arg_lst = [modpath] + arg_lst
693 693 try:
694 694 fpath = None # initialize to make sure fpath is in scope later
695 695 fpath = arg_lst[0]
696 696 filename = file_finder(fpath)
697 697 except IndexError as e:
698 698 msg = 'you must provide at least a filename.'
699 699 raise Exception(msg) from e
700 700 except IOError as e:
701 701 try:
702 702 msg = str(e)
703 703 except UnicodeError:
704 704 msg = e.message
705 705 if os.name == 'nt' and re.match(r"^'.*'$",fpath):
706 706 warn('For Windows, use double quotes to wrap a filename: %run "mypath\\myfile.py"')
707 707 raise Exception(msg) from e
708 708 except TypeError:
709 709 if fpath in sys.meta_path:
710 710 filename = ""
711 711 else:
712 712 raise
713 713
714 714 if filename.lower().endswith(('.ipy', '.ipynb')):
715 715 with preserve_keys(self.shell.user_ns, '__file__'):
716 716 self.shell.user_ns['__file__'] = filename
717 717 self.shell.safe_execfile_ipy(filename, raise_exceptions=True)
718 718 return
719 719
720 720 # Control the response to exit() calls made by the script being run
721 721 exit_ignore = 'e' in opts
722 722
723 723 # Make sure that the running script gets a proper sys.argv as if it
724 724 # were run from a system shell.
725 725 save_argv = sys.argv # save it for later restoring
726 726
727 727 if 'G' in opts:
728 728 args = arg_lst[1:]
729 729 else:
730 730 # tilde and glob expansion
731 731 args = shellglob(map(os.path.expanduser, arg_lst[1:]))
732 732
733 733 sys.argv = [filename] + args # put in the proper filename
734 734
735 735 if 'n' in opts:
736 736 name = Path(filename).stem
737 737 else:
738 738 name = '__main__'
739 739
740 740 if 'i' in opts:
741 741 # Run in user's interactive namespace
742 742 prog_ns = self.shell.user_ns
743 743 __name__save = self.shell.user_ns['__name__']
744 744 prog_ns['__name__'] = name
745 745 main_mod = self.shell.user_module
746 746
747 747 # Since '%run foo' emulates 'python foo.py' at the cmd line, we must
748 748 # set the __file__ global in the script's namespace
749 749 # TK: Is this necessary in interactive mode?
750 750 prog_ns['__file__'] = filename
751 751 else:
752 752 # Run in a fresh, empty namespace
753 753
754 754 # The shell MUST hold a reference to prog_ns so after %run
755 755 # exits, the python deletion mechanism doesn't zero it out
756 756 # (leaving dangling references). See interactiveshell for details
757 757 main_mod = self.shell.new_main_mod(filename, name)
758 758 prog_ns = main_mod.__dict__
759 759
760 760 # pickle fix. See interactiveshell for an explanation. But we need to
761 761 # make sure that, if we overwrite __main__, we replace it at the end
762 762 main_mod_name = prog_ns['__name__']
763 763
764 764 if main_mod_name == '__main__':
765 765 restore_main = sys.modules['__main__']
766 766 else:
767 767 restore_main = False
768 768
769 769 # This needs to be undone at the end to prevent holding references to
770 770 # every single object ever created.
771 771 sys.modules[main_mod_name] = main_mod
772 772
773 773 if 'p' in opts or 'd' in opts:
774 774 if 'm' in opts:
775 775 code = 'run_module(modulename, prog_ns)'
776 776 code_ns = {
777 777 'run_module': self.shell.safe_run_module,
778 778 'prog_ns': prog_ns,
779 779 'modulename': modulename,
780 780 }
781 781 else:
782 782 if 'd' in opts:
783 783 # allow exceptions to raise in debug mode
784 784 code = 'execfile(filename, prog_ns, raise_exceptions=True)'
785 785 else:
786 786 code = 'execfile(filename, prog_ns)'
787 787 code_ns = {
788 788 'execfile': self.shell.safe_execfile,
789 789 'prog_ns': prog_ns,
790 790 'filename': get_py_filename(filename),
791 791 }
792 792
793 793 try:
794 794 stats = None
795 795 if 'p' in opts:
796 796 stats = self._run_with_profiler(code, opts, code_ns)
797 797 else:
798 798 if 'd' in opts:
799 799 bp_file, bp_line = parse_breakpoint(
800 800 opts.get('b', ['1'])[0], filename)
801 801 self._run_with_debugger(
802 802 code, code_ns, filename, bp_line, bp_file)
803 803 else:
804 804 if 'm' in opts:
805 805 def run():
806 806 self.shell.safe_run_module(modulename, prog_ns)
807 807 else:
808 808 if runner is None:
809 809 runner = self.default_runner
810 810 if runner is None:
811 811 runner = self.shell.safe_execfile
812 812
813 813 def run():
814 814 runner(filename, prog_ns, prog_ns,
815 815 exit_ignore=exit_ignore)
816 816
817 817 if 't' in opts:
818 818 # timed execution
819 819 try:
820 820 nruns = int(opts['N'][0])
821 821 if nruns < 1:
822 822 error('Number of runs must be >=1')
823 823 return
824 824 except (KeyError):
825 825 nruns = 1
826 826 self._run_with_timing(run, nruns)
827 827 else:
828 828 # regular execution
829 829 run()
830 830
831 831 if 'i' in opts:
832 832 self.shell.user_ns['__name__'] = __name__save
833 833 else:
834 834 # update IPython interactive namespace
835 835
836 836 # Some forms of read errors on the file may mean the
837 837 # __name__ key was never set; using pop we don't have to
838 838 # worry about a possible KeyError.
839 839 prog_ns.pop('__name__', None)
840 840
841 841 with preserve_keys(self.shell.user_ns, '__file__'):
842 842 self.shell.user_ns.update(prog_ns)
843 843 finally:
844 844 # It's a bit of a mystery why, but __builtins__ can change from
845 845 # being a module to becoming a dict missing some key data after
846 846 # %run. As best I can see, this is NOT something IPython is doing
847 847 # at all, and similar problems have been reported before:
848 848 # http://coding.derkeiler.com/Archive/Python/comp.lang.python/2004-10/0188.html
849 849 # Since this seems to be done by the interpreter itself, the best
850 850 # we can do is to at least restore __builtins__ for the user on
851 851 # exit.
852 852 self.shell.user_ns['__builtins__'] = builtin_mod
853 853
854 854 # Ensure key global structures are restored
855 855 sys.argv = save_argv
856 856 if restore_main:
857 857 sys.modules['__main__'] = restore_main
858 858 if '__mp_main__' in sys.modules:
859 859 sys.modules['__mp_main__'] = restore_main
860 860 else:
861 861 # Remove from sys.modules the reference to main_mod we'd
862 862 # added. Otherwise it will trap references to objects
863 863 # contained therein.
864 864 del sys.modules[main_mod_name]
865 865
866 866 return stats
867 867
868 868 def _run_with_debugger(self, code, code_ns, filename=None,
869 869 bp_line=None, bp_file=None):
870 870 """
871 871 Run `code` in debugger with a break point.
872 872
873 873 Parameters
874 874 ----------
875 875 code : str
876 876 Code to execute.
877 877 code_ns : dict
878 878 A namespace in which `code` is executed.
879 879 filename : str
880 880 `code` is ran as if it is in `filename`.
881 881 bp_line : int, optional
882 882 Line number of the break point.
883 883 bp_file : str, optional
884 884 Path to the file in which break point is specified.
885 885 `filename` is used if not given.
886 886
887 887 Raises
888 888 ------
889 889 UsageError
890 890 If the break point given by `bp_line` is not valid.
891 891
892 892 """
893 893 deb = self.shell.InteractiveTB.pdb
894 894 if not deb:
895 895 self.shell.InteractiveTB.pdb = self.shell.InteractiveTB.debugger_cls()
896 896 deb = self.shell.InteractiveTB.pdb
897 897
898 898 # deb.checkline() fails if deb.curframe exists but is None; it can
899 899 # handle it not existing. https://github.com/ipython/ipython/issues/10028
900 900 if hasattr(deb, 'curframe'):
901 901 del deb.curframe
902 902
903 903 # reset Breakpoint state, which is moronically kept
904 904 # in a class
905 905 bdb.Breakpoint.next = 1
906 906 bdb.Breakpoint.bplist = {}
907 907 bdb.Breakpoint.bpbynumber = [None]
908 908 deb.clear_all_breaks()
909 909 if bp_line is not None:
910 910 # Set an initial breakpoint to stop execution
911 911 maxtries = 10
912 912 bp_file = bp_file or filename
913 913 checkline = deb.checkline(bp_file, bp_line)
914 914 if not checkline:
915 915 for bp in range(bp_line + 1, bp_line + maxtries + 1):
916 916 if deb.checkline(bp_file, bp):
917 917 break
918 918 else:
919 919 msg = ("\nI failed to find a valid line to set "
920 920 "a breakpoint\n"
921 921 "after trying up to line: %s.\n"
922 922 "Please set a valid breakpoint manually "
923 923 "with the -b option." % bp)
924 924 raise UsageError(msg)
925 925 # if we find a good linenumber, set the breakpoint
926 926 deb.do_break('%s:%s' % (bp_file, bp_line))
927 927
928 928 if filename:
929 929 # Mimic Pdb._runscript(...)
930 930 deb._wait_for_mainpyfile = True
931 931 deb.mainpyfile = deb.canonic(filename)
932 932
933 933 # Start file run
934 934 print("NOTE: Enter 'c' at the %s prompt to continue execution." % deb.prompt)
935 935 try:
936 936 if filename:
937 937 # save filename so it can be used by methods on the deb object
938 938 deb._exec_filename = filename
939 939 while True:
940 940 try:
941 941 trace = sys.gettrace()
942 942 deb.run(code, code_ns)
943 943 except Restart:
944 944 print("Restarting")
945 945 if filename:
946 946 deb._wait_for_mainpyfile = True
947 947 deb.mainpyfile = deb.canonic(filename)
948 948 continue
949 949 else:
950 950 break
951 951 finally:
952 952 sys.settrace(trace)
953 953
954 954
955 955 except:
956 956 etype, value, tb = sys.exc_info()
957 957 # Skip three frames in the traceback: the %run one,
958 958 # one inside bdb.py, and the command-line typed by the
959 959 # user (run by exec in pdb itself).
960 960 self.shell.InteractiveTB(etype, value, tb, tb_offset=3)
961 961
962 962 @staticmethod
963 963 def _run_with_timing(run, nruns):
964 964 """
965 965 Run function `run` and print timing information.
966 966
967 967 Parameters
968 968 ----------
969 969 run : callable
970 970 Any callable object which takes no argument.
971 971 nruns : int
972 972 Number of times to execute `run`.
973 973
974 974 """
975 975 twall0 = time.perf_counter()
976 976 if nruns == 1:
977 977 t0 = clock2()
978 978 run()
979 979 t1 = clock2()
980 980 t_usr = t1[0] - t0[0]
981 981 t_sys = t1[1] - t0[1]
982 982 print("\nIPython CPU timings (estimated):")
983 983 print(" User : %10.2f s." % t_usr)
984 984 print(" System : %10.2f s." % t_sys)
985 985 else:
986 986 runs = range(nruns)
987 987 t0 = clock2()
988 988 for nr in runs:
989 989 run()
990 990 t1 = clock2()
991 991 t_usr = t1[0] - t0[0]
992 992 t_sys = t1[1] - t0[1]
993 993 print("\nIPython CPU timings (estimated):")
994 994 print("Total runs performed:", nruns)
995 995 print(" Times : %10s %10s" % ('Total', 'Per run'))
996 996 print(" User : %10.2f s, %10.2f s." % (t_usr, t_usr / nruns))
997 997 print(" System : %10.2f s, %10.2f s." % (t_sys, t_sys / nruns))
998 998 twall1 = time.perf_counter()
999 999 print("Wall time: %10.2f s." % (twall1 - twall0))
1000 1000
1001 1001 @skip_doctest
1002 1002 @no_var_expand
1003 1003 @line_cell_magic
1004 1004 @needs_local_scope
1005 1005 def timeit(self, line='', cell=None, local_ns=None):
1006 1006 """Time execution of a Python statement or expression
1007 1007
1008 1008 Usage, in line mode:
1009 1009 %timeit [-n<N> -r<R> [-t|-c] -q -p<P> -o] statement
1010 1010 or in cell mode:
1011 1011 %%timeit [-n<N> -r<R> [-t|-c] -q -p<P> -o] setup_code
1012 1012 code
1013 1013 code...
1014 1014
1015 1015 Time execution of a Python statement or expression using the timeit
1016 1016 module. This function can be used both as a line and cell magic:
1017 1017
1018 1018 - In line mode you can time a single-line statement (though multiple
1019 1019 ones can be chained with using semicolons).
1020 1020
1021 1021 - In cell mode, the statement in the first line is used as setup code
1022 1022 (executed but not timed) and the body of the cell is timed. The cell
1023 1023 body has access to any variables created in the setup code.
1024 1024
1025 1025 Options:
1026 1026 -n<N>: execute the given statement <N> times in a loop. If <N> is not
1027 1027 provided, <N> is determined so as to get sufficient accuracy.
1028 1028
1029 1029 -r<R>: number of repeats <R>, each consisting of <N> loops, and take the
1030 1030 best result.
1031 1031 Default: 7
1032 1032
1033 1033 -t: use time.time to measure the time, which is the default on Unix.
1034 1034 This function measures wall time.
1035 1035
1036 1036 -c: use time.clock to measure the time, which is the default on
1037 1037 Windows and measures wall time. On Unix, resource.getrusage is used
1038 1038 instead and returns the CPU user time.
1039 1039
1040 1040 -p<P>: use a precision of <P> digits to display the timing result.
1041 1041 Default: 3
1042 1042
1043 1043 -q: Quiet, do not print result.
1044 1044
1045 1045 -o: return a TimeitResult that can be stored in a variable to inspect
1046 1046 the result in more details.
1047 1047
1048 1048 .. versionchanged:: 7.3
1049 1049 User variables are no longer expanded,
1050 1050 the magic line is always left unmodified.
1051 1051
1052 1052 Examples
1053 1053 --------
1054 1054 ::
1055 1055
1056 1056 In [1]: %timeit pass
1057 1057 8.26 ns ± 0.12 ns per loop (mean ± std. dev. of 7 runs, 100000000 loops each)
1058 1058
1059 1059 In [2]: u = None
1060 1060
1061 1061 In [3]: %timeit u is None
1062 1062 29.9 ns ± 0.643 ns per loop (mean ± std. dev. of 7 runs, 10000000 loops each)
1063 1063
1064 1064 In [4]: %timeit -r 4 u == None
1065 1065
1066 1066 In [5]: import time
1067 1067
1068 1068 In [6]: %timeit -n1 time.sleep(2)
1069 1069
1070 1070 The times reported by %timeit will be slightly higher than those
1071 1071 reported by the timeit.py script when variables are accessed. This is
1072 1072 due to the fact that %timeit executes the statement in the namespace
1073 1073 of the shell, compared with timeit.py, which uses a single setup
1074 1074 statement to import function or create variables. Generally, the bias
1075 1075 does not matter as long as results from timeit.py are not mixed with
1076 1076 those from %timeit."""
1077 1077
1078 1078 opts, stmt = self.parse_options(
1079 1079 line, "n:r:tcp:qo", posix=False, strict=False, preserve_non_opts=True
1080 1080 )
1081 1081 if stmt == "" and cell is None:
1082 1082 return
1083 1083
1084 1084 timefunc = timeit.default_timer
1085 1085 number = int(getattr(opts, "n", 0))
1086 1086 default_repeat = 7 if timeit.default_repeat < 7 else timeit.default_repeat
1087 1087 repeat = int(getattr(opts, "r", default_repeat))
1088 1088 precision = int(getattr(opts, "p", 3))
1089 1089 quiet = 'q' in opts
1090 1090 return_result = 'o' in opts
1091 1091 if hasattr(opts, "t"):
1092 1092 timefunc = time.time
1093 1093 if hasattr(opts, "c"):
1094 1094 timefunc = clock
1095 1095
1096 1096 timer = Timer(timer=timefunc)
1097 1097 # this code has tight coupling to the inner workings of timeit.Timer,
1098 1098 # but is there a better way to achieve that the code stmt has access
1099 1099 # to the shell namespace?
1100 1100 transform = self.shell.transform_cell
1101 1101
1102 1102 if cell is None:
1103 1103 # called as line magic
1104 1104 ast_setup = self.shell.compile.ast_parse("pass")
1105 1105 ast_stmt = self.shell.compile.ast_parse(transform(stmt))
1106 1106 else:
1107 1107 ast_setup = self.shell.compile.ast_parse(transform(stmt))
1108 1108 ast_stmt = self.shell.compile.ast_parse(transform(cell))
1109 1109
1110 1110 ast_setup = self.shell.transform_ast(ast_setup)
1111 1111 ast_stmt = self.shell.transform_ast(ast_stmt)
1112 1112
1113 1113 # Check that these compile to valid Python code *outside* the timer func
1114 1114 # Invalid code may become valid when put inside the function & loop,
1115 1115 # which messes up error messages.
1116 1116 # https://github.com/ipython/ipython/issues/10636
1117 1117 self.shell.compile(ast_setup, "<magic-timeit-setup>", "exec")
1118 1118 self.shell.compile(ast_stmt, "<magic-timeit-stmt>", "exec")
1119 1119
1120 1120 # This codestring is taken from timeit.template - we fill it in as an
1121 1121 # AST, so that we can apply our AST transformations to the user code
1122 1122 # without affecting the timing code.
1123 1123 timeit_ast_template = ast.parse('def inner(_it, _timer):\n'
1124 1124 ' setup\n'
1125 1125 ' _t0 = _timer()\n'
1126 1126 ' for _i in _it:\n'
1127 1127 ' stmt\n'
1128 1128 ' _t1 = _timer()\n'
1129 1129 ' return _t1 - _t0\n')
1130 1130
1131 1131 timeit_ast = TimeitTemplateFiller(ast_setup, ast_stmt).visit(timeit_ast_template)
1132 1132 timeit_ast = ast.fix_missing_locations(timeit_ast)
1133 1133
1134 1134 # Track compilation time so it can be reported if too long
1135 1135 # Minimum time above which compilation time will be reported
1136 1136 tc_min = 0.1
1137 1137
1138 1138 t0 = clock()
1139 1139 code = self.shell.compile(timeit_ast, "<magic-timeit>", "exec")
1140 1140 tc = clock()-t0
1141 1141
1142 1142 ns = {}
1143 1143 glob = self.shell.user_ns
1144 1144 # handles global vars with same name as local vars. We store them in conflict_globs.
1145 1145 conflict_globs = {}
1146 1146 if local_ns and cell is None:
1147 1147 for var_name, var_val in glob.items():
1148 1148 if var_name in local_ns:
1149 1149 conflict_globs[var_name] = var_val
1150 1150 glob.update(local_ns)
1151 1151
1152 1152 exec(code, glob, ns)
1153 1153 timer.inner = ns["inner"]
1154 1154
1155 1155 # This is used to check if there is a huge difference between the
1156 1156 # best and worst timings.
1157 1157 # Issue: https://github.com/ipython/ipython/issues/6471
1158 1158 if number == 0:
1159 1159 # determine number so that 0.2 <= total time < 2.0
1160 1160 for index in range(0, 10):
1161 1161 number = 10 ** index
1162 1162 time_number = timer.timeit(number)
1163 1163 if time_number >= 0.2:
1164 1164 break
1165 1165
1166 1166 all_runs = timer.repeat(repeat, number)
1167 1167 best = min(all_runs) / number
1168 1168 worst = max(all_runs) / number
1169 1169 timeit_result = TimeitResult(number, repeat, best, worst, all_runs, tc, precision)
1170 1170
1171 1171 # Restore global vars from conflict_globs
1172 1172 if conflict_globs:
1173 1173 glob.update(conflict_globs)
1174 1174
1175 1175 if not quiet :
1176 1176 # Check best timing is greater than zero to avoid a
1177 1177 # ZeroDivisionError.
1178 1178 # In cases where the slowest timing is lesser than a microsecond
1179 1179 # we assume that it does not really matter if the fastest
1180 1180 # timing is 4 times faster than the slowest timing or not.
1181 1181 if worst > 4 * best and best > 0 and worst > 1e-6:
1182 1182 print("The slowest run took %0.2f times longer than the "
1183 1183 "fastest. This could mean that an intermediate result "
1184 1184 "is being cached." % (worst / best))
1185 1185
1186 1186 print( timeit_result )
1187 1187
1188 1188 if tc > tc_min:
1189 1189 print("Compiler time: %.2f s" % tc)
1190 1190 if return_result:
1191 1191 return timeit_result
1192 1192
1193 1193 @skip_doctest
1194 1194 @no_var_expand
1195 1195 @needs_local_scope
1196 1196 @line_cell_magic
1197 1197 def time(self,line='', cell=None, local_ns=None):
1198 1198 """Time execution of a Python statement or expression.
1199 1199
1200 1200 The CPU and wall clock times are printed, and the value of the
1201 1201 expression (if any) is returned. Note that under Win32, system time
1202 1202 is always reported as 0, since it can not be measured.
1203 1203
1204 1204 This function can be used both as a line and cell magic:
1205 1205
1206 1206 - In line mode you can time a single-line statement (though multiple
1207 1207 ones can be chained with using semicolons).
1208 1208
1209 1209 - In cell mode, you can time the cell body (a directly
1210 1210 following statement raises an error).
1211 1211
1212 1212 This function provides very basic timing functionality. Use the timeit
1213 1213 magic for more control over the measurement.
1214 1214
1215 1215 .. versionchanged:: 7.3
1216 1216 User variables are no longer expanded,
1217 1217 the magic line is always left unmodified.
1218 1218
1219 1219 Examples
1220 1220 --------
1221 1221 ::
1222 1222
1223 1223 In [1]: %time 2**128
1224 1224 CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
1225 1225 Wall time: 0.00
1226 1226 Out[1]: 340282366920938463463374607431768211456L
1227 1227
1228 1228 In [2]: n = 1000000
1229 1229
1230 1230 In [3]: %time sum(range(n))
1231 1231 CPU times: user 1.20 s, sys: 0.05 s, total: 1.25 s
1232 1232 Wall time: 1.37
1233 1233 Out[3]: 499999500000L
1234 1234
1235 1235 In [4]: %time print 'hello world'
1236 1236 hello world
1237 1237 CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
1238 1238 Wall time: 0.00
1239 1239
1240 1240 .. note::
1241 1241 The time needed by Python to compile the given expression will be
1242 1242 reported if it is more than 0.1s.
1243 1243
1244 1244 In the example below, the actual exponentiation is done by Python
1245 1245 at compilation time, so while the expression can take a noticeable
1246 1246 amount of time to compute, that time is purely due to the
1247 1247 compilation::
1248 1248
1249 1249 In [5]: %time 3**9999;
1250 1250 CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
1251 1251 Wall time: 0.00 s
1252 1252
1253 1253 In [6]: %time 3**999999;
1254 1254 CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
1255 1255 Wall time: 0.00 s
1256 1256 Compiler : 0.78 s
1257 1257 """
1258 1258 # fail immediately if the given expression can't be compiled
1259 1259
1260 1260 if line and cell:
1261 1261 raise UsageError("Can't use statement directly after '%%time'!")
1262 1262
1263 1263 if cell:
1264 1264 expr = self.shell.transform_cell(cell)
1265 1265 else:
1266 1266 expr = self.shell.transform_cell(line)
1267 1267
1268 1268 # Minimum time above which parse time will be reported
1269 1269 tp_min = 0.1
1270 1270
1271 1271 t0 = clock()
1272 1272 expr_ast = self.shell.compile.ast_parse(expr)
1273 1273 tp = clock()-t0
1274 1274
1275 1275 # Apply AST transformations
1276 1276 expr_ast = self.shell.transform_ast(expr_ast)
1277 1277
1278 1278 # Minimum time above which compilation time will be reported
1279 1279 tc_min = 0.1
1280 1280
1281 1281 expr_val=None
1282 1282 if len(expr_ast.body)==1 and isinstance(expr_ast.body[0], ast.Expr):
1283 1283 mode = 'eval'
1284 1284 source = '<timed eval>'
1285 1285 expr_ast = ast.Expression(expr_ast.body[0].value)
1286 1286 else:
1287 1287 mode = 'exec'
1288 1288 source = '<timed exec>'
1289 1289 # multi-line %%time case
1290 1290 if len(expr_ast.body) > 1 and isinstance(expr_ast.body[-1], ast.Expr):
1291 1291 expr_val= expr_ast.body[-1]
1292 1292 expr_ast = expr_ast.body[:-1]
1293 1293 expr_ast = Module(expr_ast, [])
1294 1294 expr_val = ast.Expression(expr_val.value)
1295 1295
1296 1296 t0 = clock()
1297 1297 code = self.shell.compile(expr_ast, source, mode)
1298 1298 tc = clock()-t0
1299 1299
1300 1300 # skew measurement as little as possible
1301 1301 glob = self.shell.user_ns
1302 1302 wtime = time.time
1303 1303 # time execution
1304 1304 wall_st = wtime()
1305 1305 if mode=='eval':
1306 1306 st = clock2()
1307 1307 try:
1308 1308 out = eval(code, glob, local_ns)
1309 1309 except:
1310 1310 self.shell.showtraceback()
1311 1311 return
1312 1312 end = clock2()
1313 1313 else:
1314 1314 st = clock2()
1315 1315 try:
1316 1316 exec(code, glob, local_ns)
1317 1317 out=None
1318 1318 # multi-line %%time case
1319 1319 if expr_val is not None:
1320 1320 code_2 = self.shell.compile(expr_val, source, 'eval')
1321 1321 out = eval(code_2, glob, local_ns)
1322 1322 except:
1323 1323 self.shell.showtraceback()
1324 1324 return
1325 1325 end = clock2()
1326 1326
1327 1327 wall_end = wtime()
1328 1328 # Compute actual times and report
1329 1329 wall_time = wall_end - wall_st
1330 1330 cpu_user = end[0] - st[0]
1331 1331 cpu_sys = end[1] - st[1]
1332 1332 cpu_tot = cpu_user + cpu_sys
1333 1333 # On windows cpu_sys is always zero, so only total is displayed
1334 1334 if sys.platform != "win32":
1335 1335 print(
1336 1336 f"CPU times: user {_format_time(cpu_user)}, sys: {_format_time(cpu_sys)}, total: {_format_time(cpu_tot)}"
1337 1337 )
1338 1338 else:
1339 1339 print(f"CPU times: total: {_format_time(cpu_tot)}")
1340 1340 print(f"Wall time: {_format_time(wall_time)}")
1341 1341 if tc > tc_min:
1342 1342 print(f"Compiler : {_format_time(tc)}")
1343 1343 if tp > tp_min:
1344 1344 print(f"Parser : {_format_time(tp)}")
1345 1345 return out
1346 1346
1347 1347 @skip_doctest
1348 1348 @line_magic
1349 1349 def macro(self, parameter_s=''):
1350 1350 """Define a macro for future re-execution. It accepts ranges of history,
1351 1351 filenames or string objects.
1352 1352
1353 1353 Usage:\\
1354 1354 %macro [options] name n1-n2 n3-n4 ... n5 .. n6 ...
1355 1355
1356 1356 Options:
1357 1357
1358 1358 -r: use 'raw' input. By default, the 'processed' history is used,
1359 1359 so that magics are loaded in their transformed version to valid
1360 1360 Python. If this option is given, the raw input as typed at the
1361 1361 command line is used instead.
1362 1362
1363 1363 -q: quiet macro definition. By default, a tag line is printed
1364 1364 to indicate the macro has been created, and then the contents of
1365 1365 the macro are printed. If this option is given, then no printout
1366 1366 is produced once the macro is created.
1367 1367
1368 1368 This will define a global variable called `name` which is a string
1369 1369 made of joining the slices and lines you specify (n1,n2,... numbers
1370 1370 above) from your input history into a single string. This variable
1371 1371 acts like an automatic function which re-executes those lines as if
1372 1372 you had typed them. You just type 'name' at the prompt and the code
1373 1373 executes.
1374 1374
1375 1375 The syntax for indicating input ranges is described in %history.
1376 1376
1377 1377 Note: as a 'hidden' feature, you can also use traditional python slice
1378 1378 notation, where N:M means numbers N through M-1.
1379 1379
1380 1380 For example, if your history contains (print using %hist -n )::
1381 1381
1382 1382 44: x=1
1383 1383 45: y=3
1384 1384 46: z=x+y
1385 1385 47: print x
1386 1386 48: a=5
1387 1387 49: print 'x',x,'y',y
1388 1388
1389 1389 you can create a macro with lines 44 through 47 (included) and line 49
1390 1390 called my_macro with::
1391 1391
1392 1392 In [55]: %macro my_macro 44-47 49
1393 1393
1394 1394 Now, typing `my_macro` (without quotes) will re-execute all this code
1395 1395 in one pass.
1396 1396
1397 1397 You don't need to give the line-numbers in order, and any given line
1398 1398 number can appear multiple times. You can assemble macros with any
1399 1399 lines from your input history in any order.
1400 1400
1401 1401 The macro is a simple object which holds its value in an attribute,
1402 1402 but IPython's display system checks for macros and executes them as
1403 1403 code instead of printing them when you type their name.
1404 1404
1405 1405 You can view a macro's contents by explicitly printing it with::
1406 1406
1407 1407 print macro_name
1408 1408
1409 1409 """
1410 1410 opts,args = self.parse_options(parameter_s,'rq',mode='list')
1411 1411 if not args: # List existing macros
1412 1412 return sorted(k for k,v in self.shell.user_ns.items() if isinstance(v, Macro))
1413 1413 if len(args) == 1:
1414 1414 raise UsageError(
1415 1415 "%macro insufficient args; usage '%macro name n1-n2 n3-4...")
1416 1416 name, codefrom = args[0], " ".join(args[1:])
1417 1417
1418 1418 #print 'rng',ranges # dbg
1419 1419 try:
1420 1420 lines = self.shell.find_user_code(codefrom, 'r' in opts)
1421 1421 except (ValueError, TypeError) as e:
1422 1422 print(e.args[0])
1423 1423 return
1424 1424 macro = Macro(lines)
1425 1425 self.shell.define_macro(name, macro)
1426 1426 if not ( 'q' in opts) :
1427 1427 print('Macro `%s` created. To execute, type its name (without quotes).' % name)
1428 1428 print('=== Macro contents: ===')
1429 1429 print(macro, end=' ')
1430 1430
1431 1431 @magic_arguments.magic_arguments()
1432 1432 @magic_arguments.argument('output', type=str, default='', nargs='?',
1433 1433 help="""The name of the variable in which to store output.
1434 1434 This is a utils.io.CapturedIO object with stdout/err attributes
1435 1435 for the text of the captured output.
1436 1436
1437 1437 CapturedOutput also has a show() method for displaying the output,
1438 1438 and __call__ as well, so you can use that to quickly display the
1439 1439 output.
1440 1440
1441 1441 If unspecified, captured output is discarded.
1442 1442 """
1443 1443 )
1444 1444 @magic_arguments.argument('--no-stderr', action="store_true",
1445 1445 help="""Don't capture stderr."""
1446 1446 )
1447 1447 @magic_arguments.argument('--no-stdout', action="store_true",
1448 1448 help="""Don't capture stdout."""
1449 1449 )
1450 1450 @magic_arguments.argument('--no-display', action="store_true",
1451 1451 help="""Don't capture IPython's rich display."""
1452 1452 )
1453 1453 @cell_magic
1454 1454 def capture(self, line, cell):
1455 1455 """run the cell, capturing stdout, stderr, and IPython's rich display() calls."""
1456 1456 args = magic_arguments.parse_argstring(self.capture, line)
1457 1457 out = not args.no_stdout
1458 1458 err = not args.no_stderr
1459 1459 disp = not args.no_display
1460 1460 with capture_output(out, err, disp) as io:
1461 1461 self.shell.run_cell(cell)
1462 1462 if args.output:
1463 1463 self.shell.user_ns[args.output] = io
1464 1464
1465 1465 def parse_breakpoint(text, current_file):
1466 1466 '''Returns (file, line) for file:line and (current_file, line) for line'''
1467 1467 colon = text.find(':')
1468 1468 if colon == -1:
1469 1469 return current_file, int(text)
1470 1470 else:
1471 1471 return text[:colon], int(text[colon+1:])
1472 1472
1473 1473 def _format_time(timespan, precision=3):
1474 1474 """Formats the timespan in a human readable form"""
1475 1475
1476 1476 if timespan >= 60.0:
1477 1477 # we have more than a minute, format that in a human readable form
1478 1478 # Idea from http://snipplr.com/view/5713/
1479 1479 parts = [("d", 60*60*24),("h", 60*60),("min", 60), ("s", 1)]
1480 1480 time = []
1481 1481 leftover = timespan
1482 1482 for suffix, length in parts:
1483 1483 value = int(leftover / length)
1484 1484 if value > 0:
1485 1485 leftover = leftover % length
1486 1486 time.append(u'%s%s' % (str(value), suffix))
1487 1487 if leftover < 1:
1488 1488 break
1489 1489 return " ".join(time)
1490 1490
1491 1491
1492 1492 # Unfortunately the unicode 'micro' symbol can cause problems in
1493 1493 # certain terminals.
1494 1494 # See bug: https://bugs.launchpad.net/ipython/+bug/348466
1495 1495 # Try to prevent crashes by being more secure than it needs to
1496 1496 # E.g. eclipse is able to print a µ, but has no sys.stdout.encoding set.
1497 1497 units = [u"s", u"ms",u'us',"ns"] # the save value
1498 1498 if hasattr(sys.stdout, 'encoding') and sys.stdout.encoding:
1499 1499 try:
1500 1500 u'\xb5'.encode(sys.stdout.encoding)
1501 1501 units = [u"s", u"ms",u'\xb5s',"ns"]
1502 1502 except:
1503 1503 pass
1504 1504 scaling = [1, 1e3, 1e6, 1e9]
1505 1505
1506 1506 if timespan > 0.0:
1507 1507 order = min(-int(math.floor(math.log10(timespan)) // 3), 3)
1508 1508 else:
1509 1509 order = 3
1510 1510 return u"%.*g %s" % (precision, timespan * scaling[order], units[order])
@@ -1,362 +1,362 b''
1 1 """Magic functions for running cells in various scripts."""
2 2
3 3 # Copyright (c) IPython Development Team.
4 4 # Distributed under the terms of the Modified BSD License.
5 5
6 6 import asyncio
7 7 import atexit
8 8 import errno
9 9 import os
10 10 import signal
11 11 import sys
12 12 import time
13 13 from subprocess import CalledProcessError
14 14 from threading import Thread
15 15
16 16 from traitlets import Any, Dict, List, default
17 17
18 18 from IPython.core import magic_arguments
19 19 from IPython.core.async_helpers import _AsyncIOProxy
20 20 from IPython.core.magic import Magics, cell_magic, line_magic, magics_class
21 21 from IPython.utils.process import arg_split
22 22
23 23 #-----------------------------------------------------------------------------
24 24 # Magic implementation classes
25 25 #-----------------------------------------------------------------------------
26 26
27 27 def script_args(f):
28 28 """single decorator for adding script args"""
29 29 args = [
30 30 magic_arguments.argument(
31 31 '--out', type=str,
32 32 help="""The variable in which to store stdout from the script.
33 33 If the script is backgrounded, this will be the stdout *pipe*,
34 34 instead of the stderr text itself and will not be auto closed.
35 35 """
36 36 ),
37 37 magic_arguments.argument(
38 38 '--err', type=str,
39 39 help="""The variable in which to store stderr from the script.
40 40 If the script is backgrounded, this will be the stderr *pipe*,
41 41 instead of the stderr text itself and will not be autoclosed.
42 42 """
43 43 ),
44 44 magic_arguments.argument(
45 45 '--bg', action="store_true",
46 46 help="""Whether to run the script in the background.
47 47 If given, the only way to see the output of the command is
48 48 with --out/err.
49 49 """
50 50 ),
51 51 magic_arguments.argument(
52 52 '--proc', type=str,
53 53 help="""The variable in which to store Popen instance.
54 54 This is used only when --bg option is given.
55 55 """
56 56 ),
57 57 magic_arguments.argument(
58 58 '--no-raise-error', action="store_false", dest='raise_error',
59 59 help="""Whether you should raise an error message in addition to
60 60 a stream on stderr if you get a nonzero exit code.
61 """
62 )
61 """,
62 ),
63 63 ]
64 64 for arg in args:
65 65 f = arg(f)
66 66 return f
67 67
68 68
69 69 @magics_class
70 70 class ScriptMagics(Magics):
71 71 """Magics for talking to scripts
72 72
73 73 This defines a base `%%script` cell magic for running a cell
74 74 with a program in a subprocess, and registers a few top-level
75 75 magics that call %%script with common interpreters.
76 76 """
77 77
78 78 event_loop = Any(
79 79 help="""
80 80 The event loop on which to run subprocesses
81 81
82 82 Not the main event loop,
83 83 because we want to be able to make blocking calls
84 84 and have certain requirements we don't want to impose on the main loop.
85 85 """
86 86 )
87 87
88 88 script_magics = List(
89 89 help="""Extra script cell magics to define
90 90
91 91 This generates simple wrappers of `%%script foo` as `%%foo`.
92 92
93 93 If you want to add script magics that aren't on your path,
94 94 specify them in script_paths
95 95 """,
96 96 ).tag(config=True)
97 97 @default('script_magics')
98 98 def _script_magics_default(self):
99 99 """default to a common list of programs"""
100 100
101 101 defaults = [
102 102 'sh',
103 103 'bash',
104 104 'perl',
105 105 'ruby',
106 106 'python',
107 107 'python2',
108 108 'python3',
109 109 'pypy',
110 110 ]
111 111 if os.name == 'nt':
112 112 defaults.extend([
113 113 'cmd',
114 114 ])
115 115
116 116 return defaults
117 117
118 118 script_paths = Dict(
119 119 help="""Dict mapping short 'ruby' names to full paths, such as '/opt/secret/bin/ruby'
120 120
121 121 Only necessary for items in script_magics where the default path will not
122 122 find the right interpreter.
123 123 """
124 124 ).tag(config=True)
125 125
126 126 def __init__(self, shell=None):
127 127 super(ScriptMagics, self).__init__(shell=shell)
128 128 self._generate_script_magics()
129 129 self.bg_processes = []
130 130 atexit.register(self.kill_bg_processes)
131 131
132 132 def __del__(self):
133 133 self.kill_bg_processes()
134 134
135 135 def _generate_script_magics(self):
136 136 cell_magics = self.magics['cell']
137 137 for name in self.script_magics:
138 138 cell_magics[name] = self._make_script_magic(name)
139 139
140 140 def _make_script_magic(self, name):
141 141 """make a named magic, that calls %%script with a particular program"""
142 142 # expand to explicit path if necessary:
143 143 script = self.script_paths.get(name, name)
144 144
145 145 @magic_arguments.magic_arguments()
146 146 @script_args
147 147 def named_script_magic(line, cell):
148 148 # if line, add it as cl-flags
149 149 if line:
150 150 line = "%s %s" % (script, line)
151 151 else:
152 152 line = script
153 153 return self.shebang(line, cell)
154 154
155 155 # write a basic docstring:
156 156 named_script_magic.__doc__ = \
157 157 """%%{name} script magic
158 158
159 159 Run cells with {script} in a subprocess.
160 160
161 161 This is a shortcut for `%%script {script}`
162 162 """.format(**locals())
163 163
164 164 return named_script_magic
165 165
166 166 @magic_arguments.magic_arguments()
167 167 @script_args
168 168 @cell_magic("script")
169 169 def shebang(self, line, cell):
170 170 """Run a cell via a shell command
171 171
172 172 The `%%script` line is like the #! line of script,
173 173 specifying a program (bash, perl, ruby, etc.) with which to run.
174 174
175 175 The rest of the cell is run by that program.
176 176
177 177 Examples
178 178 --------
179 179 ::
180 180
181 181 In [1]: %%script bash
182 182 ...: for i in 1 2 3; do
183 183 ...: echo $i
184 184 ...: done
185 185 1
186 186 2
187 187 3
188 188 """
189 189
190 190 # Create the event loop in which to run script magics
191 191 # this operates on a background thread
192 192 if self.event_loop is None:
193 193 if sys.platform == "win32":
194 194 # don't override the current policy,
195 195 # just create an event loop
196 196 event_loop = asyncio.WindowsProactorEventLoopPolicy().new_event_loop()
197 197 else:
198 198 event_loop = asyncio.new_event_loop()
199 199 self.event_loop = event_loop
200 200
201 201 # start the loop in a background thread
202 202 asyncio_thread = Thread(target=event_loop.run_forever, daemon=True)
203 203 asyncio_thread.start()
204 204 else:
205 205 event_loop = self.event_loop
206 206
207 207 def in_thread(coro):
208 208 """Call a coroutine on the asyncio thread"""
209 209 return asyncio.run_coroutine_threadsafe(coro, event_loop).result()
210 210
211 211 async def _handle_stream(stream, stream_arg, file_object):
212 212 while True:
213 213 line = (await stream.readline()).decode("utf8")
214 214 if not line:
215 215 break
216 216 if stream_arg:
217 217 self.shell.user_ns[stream_arg] = line
218 218 else:
219 219 file_object.write(line)
220 220 file_object.flush()
221 221
222 222 async def _stream_communicate(process, cell):
223 223 process.stdin.write(cell)
224 224 process.stdin.close()
225 225 stdout_task = asyncio.create_task(
226 226 _handle_stream(process.stdout, args.out, sys.stdout)
227 227 )
228 228 stderr_task = asyncio.create_task(
229 229 _handle_stream(process.stderr, args.err, sys.stderr)
230 230 )
231 231 await asyncio.wait([stdout_task, stderr_task])
232 232 await process.wait()
233 233
234 234 argv = arg_split(line, posix=not sys.platform.startswith("win"))
235 235 args, cmd = self.shebang.parser.parse_known_args(argv)
236 236
237 237 try:
238 238 p = in_thread(
239 239 asyncio.create_subprocess_exec(
240 240 *cmd,
241 241 stdout=asyncio.subprocess.PIPE,
242 242 stderr=asyncio.subprocess.PIPE,
243 243 stdin=asyncio.subprocess.PIPE,
244 244 )
245 245 )
246 246 except OSError as e:
247 247 if e.errno == errno.ENOENT:
248 248 print("Couldn't find program: %r" % cmd[0])
249 249 return
250 250 else:
251 251 raise
252 252
253 253 if not cell.endswith('\n'):
254 254 cell += '\n'
255 255 cell = cell.encode('utf8', 'replace')
256 256 if args.bg:
257 257 self.bg_processes.append(p)
258 258 self._gc_bg_processes()
259 259 to_close = []
260 260 if args.out:
261 261 self.shell.user_ns[args.out] = _AsyncIOProxy(p.stdout, event_loop)
262 262 else:
263 263 to_close.append(p.stdout)
264 264 if args.err:
265 265 self.shell.user_ns[args.err] = _AsyncIOProxy(p.stderr, event_loop)
266 266 else:
267 267 to_close.append(p.stderr)
268 268 event_loop.call_soon_threadsafe(
269 269 lambda: asyncio.Task(self._run_script(p, cell, to_close))
270 270 )
271 271 if args.proc:
272 272 proc_proxy = _AsyncIOProxy(p, event_loop)
273 273 proc_proxy.stdout = _AsyncIOProxy(p.stdout, event_loop)
274 274 proc_proxy.stderr = _AsyncIOProxy(p.stderr, event_loop)
275 275 self.shell.user_ns[args.proc] = proc_proxy
276 276 return
277 277
278 278 try:
279 279 in_thread(_stream_communicate(p, cell))
280 280 except KeyboardInterrupt:
281 281 try:
282 282 p.send_signal(signal.SIGINT)
283 283 in_thread(asyncio.wait_for(p.wait(), timeout=0.1))
284 284 if p.returncode is not None:
285 285 print("Process is interrupted.")
286 286 return
287 287 p.terminate()
288 288 in_thread(asyncio.wait_for(p.wait(), timeout=0.1))
289 289 if p.returncode is not None:
290 290 print("Process is terminated.")
291 291 return
292 292 p.kill()
293 293 print("Process is killed.")
294 294 except OSError:
295 295 pass
296 296 except Exception as e:
297 297 print("Error while terminating subprocess (pid=%i): %s" % (p.pid, e))
298 298 return
299 299
300 300 if args.raise_error and p.returncode != 0:
301 301 # If we get here and p.returncode is still None, we must have
302 302 # killed it but not yet seen its return code. We don't wait for it,
303 303 # in case it's stuck in uninterruptible sleep. -9 = SIGKILL
304 304 rc = p.returncode or -9
305 305 raise CalledProcessError(rc, cell)
306 306
307 307 shebang.__skip_doctest__ = os.name != "posix"
308 308
309 309 async def _run_script(self, p, cell, to_close):
310 310 """callback for running the script in the background"""
311 311
312 312 p.stdin.write(cell)
313 313 await p.stdin.drain()
314 314 p.stdin.close()
315 315 await p.stdin.wait_closed()
316 316 await p.wait()
317 317 # asyncio read pipes have no close
318 318 # but we should drain the data anyway
319 319 for s in to_close:
320 320 await s.read()
321 321 self._gc_bg_processes()
322 322
323 323 @line_magic("killbgscripts")
324 324 def killbgscripts(self, _nouse_=''):
325 325 """Kill all BG processes started by %%script and its family."""
326 326 self.kill_bg_processes()
327 327 print("All background processes were killed.")
328 328
329 329 def kill_bg_processes(self):
330 330 """Kill all BG processes which are still running."""
331 331 if not self.bg_processes:
332 332 return
333 333 for p in self.bg_processes:
334 334 if p.returncode is None:
335 335 try:
336 336 p.send_signal(signal.SIGINT)
337 337 except:
338 338 pass
339 339 time.sleep(0.1)
340 340 self._gc_bg_processes()
341 341 if not self.bg_processes:
342 342 return
343 343 for p in self.bg_processes:
344 344 if p.returncode is None:
345 345 try:
346 346 p.terminate()
347 347 except:
348 348 pass
349 349 time.sleep(0.1)
350 350 self._gc_bg_processes()
351 351 if not self.bg_processes:
352 352 return
353 353 for p in self.bg_processes:
354 354 if p.returncode is None:
355 355 try:
356 356 p.kill()
357 357 except:
358 358 pass
359 359 self._gc_bg_processes()
360 360
361 361 def _gc_bg_processes(self):
362 362 self.bg_processes = [p for p in self.bg_processes if p.returncode is None]
@@ -1,442 +1,443 b''
1 1 """Tests for the token-based transformers in IPython.core.inputtransformer2
2 2
3 3 Line-based transformers are the simpler ones; token-based transformers are
4 4 more complex. See test_inputtransformer2_line for tests for line-based
5 5 transformations.
6 6 """
7 7 import platform
8 8 import string
9 9 import sys
10 10 from textwrap import dedent
11 11
12 12 import pytest
13 13
14 14 from IPython.core import inputtransformer2 as ipt2
15 15 from IPython.core.inputtransformer2 import _find_assign_op, make_tokens_by_line
16 16
17 17 MULTILINE_MAGIC = (
18 18 """\
19 19 a = f()
20 20 %foo \\
21 21 bar
22 22 g()
23 23 """.splitlines(
24 24 keepends=True
25 25 ),
26 26 (2, 0),
27 27 """\
28 28 a = f()
29 29 get_ipython().run_line_magic('foo', ' bar')
30 30 g()
31 31 """.splitlines(
32 32 keepends=True
33 33 ),
34 34 )
35 35
36 36 INDENTED_MAGIC = (
37 37 """\
38 38 for a in range(5):
39 39 %ls
40 40 """.splitlines(
41 41 keepends=True
42 42 ),
43 43 (2, 4),
44 44 """\
45 45 for a in range(5):
46 46 get_ipython().run_line_magic('ls', '')
47 47 """.splitlines(
48 48 keepends=True
49 49 ),
50 50 )
51 51
52 52 CRLF_MAGIC = (
53 53 ["a = f()\n", "%ls\r\n", "g()\n"],
54 54 (2, 0),
55 55 ["a = f()\n", "get_ipython().run_line_magic('ls', '')\n", "g()\n"],
56 56 )
57 57
58 58 MULTILINE_MAGIC_ASSIGN = (
59 59 """\
60 60 a = f()
61 61 b = %foo \\
62 62 bar
63 63 g()
64 64 """.splitlines(
65 65 keepends=True
66 66 ),
67 67 (2, 4),
68 68 """\
69 69 a = f()
70 70 b = get_ipython().run_line_magic('foo', ' bar')
71 71 g()
72 72 """.splitlines(
73 73 keepends=True
74 74 ),
75 75 )
76 76
77 77 MULTILINE_SYSTEM_ASSIGN = ("""\
78 78 a = f()
79 79 b = !foo \\
80 80 bar
81 81 g()
82 82 """.splitlines(keepends=True), (2, 4), """\
83 83 a = f()
84 84 b = get_ipython().getoutput('foo bar')
85 85 g()
86 86 """.splitlines(keepends=True))
87 87
88 88 #####
89 89
90 MULTILINE_SYSTEM_ASSIGN_AFTER_DEDENT = ("""\
90 MULTILINE_SYSTEM_ASSIGN_AFTER_DEDENT = (
91 """\
91 92 def test():
92 93 for i in range(1):
93 94 print(i)
94 95 res =! ls
95 96 """.splitlines(
96 97 keepends=True
97 98 ),
98 99 (4, 7),
99 100 """\
100 101 def test():
101 102 for i in range(1):
102 103 print(i)
103 104 res =get_ipython().getoutput(\' ls\')
104 105 """.splitlines(
105 106 keepends=True
106 107 ),
107 108 )
108 109
109 110 ######
110 111
111 112 AUTOCALL_QUOTE = ([",f 1 2 3\n"], (1, 0), ['f("1", "2", "3")\n'])
112 113
113 114 AUTOCALL_QUOTE2 = ([";f 1 2 3\n"], (1, 0), ['f("1 2 3")\n'])
114 115
115 116 AUTOCALL_PAREN = (["/f 1 2 3\n"], (1, 0), ["f(1, 2, 3)\n"])
116 117
117 118 SIMPLE_HELP = (["foo?\n"], (1, 0), ["get_ipython().run_line_magic('pinfo', 'foo')\n"])
118 119
119 120 DETAILED_HELP = (
120 121 ["foo??\n"],
121 122 (1, 0),
122 123 ["get_ipython().run_line_magic('pinfo2', 'foo')\n"],
123 124 )
124 125
125 126 MAGIC_HELP = (["%foo?\n"], (1, 0), ["get_ipython().run_line_magic('pinfo', '%foo')\n"])
126 127
127 128 HELP_IN_EXPR = (
128 129 ["a = b + c?\n"],
129 130 (1, 0),
130 131 ["get_ipython().run_line_magic('pinfo', 'c')\n"],
131 132 )
132 133
133 134 HELP_CONTINUED_LINE = (
134 135 """\
135 136 a = \\
136 137 zip?
137 138 """.splitlines(
138 139 keepends=True
139 140 ),
140 141 (1, 0),
141 142 [r"get_ipython().run_line_magic('pinfo', 'zip')" + "\n"],
142 143 )
143 144
144 145 HELP_MULTILINE = (
145 146 """\
146 147 (a,
147 148 b) = zip?
148 149 """.splitlines(
149 150 keepends=True
150 151 ),
151 152 (1, 0),
152 153 [r"get_ipython().run_line_magic('pinfo', 'zip')" + "\n"],
153 154 )
154 155
155 156 HELP_UNICODE = (
156 157 ["π.foo?\n"],
157 158 (1, 0),
158 159 ["get_ipython().run_line_magic('pinfo', 'π.foo')\n"],
159 160 )
160 161
161 162
162 163 def null_cleanup_transformer(lines):
163 164 """
164 165 A cleanup transform that returns an empty list.
165 166 """
166 167 return []
167 168
168 169
169 170 def test_check_make_token_by_line_never_ends_empty():
170 171 """
171 172 Check that not sequence of single or double characters ends up leading to en empty list of tokens
172 173 """
173 174 from string import printable
174 175
175 176 for c in printable:
176 177 assert make_tokens_by_line(c)[-1] != []
177 178 for k in printable:
178 179 assert make_tokens_by_line(c + k)[-1] != []
179 180
180 181
181 182 def check_find(transformer, case, match=True):
182 183 sample, expected_start, _ = case
183 184 tbl = make_tokens_by_line(sample)
184 185 res = transformer.find(tbl)
185 186 if match:
186 187 # start_line is stored 0-indexed, expected values are 1-indexed
187 188 assert (res.start_line + 1, res.start_col) == expected_start
188 189 return res
189 190 else:
190 191 assert res is None
191 192
192 193
193 194 def check_transform(transformer_cls, case):
194 195 lines, start, expected = case
195 196 transformer = transformer_cls(start)
196 197 assert transformer.transform(lines) == expected
197 198
198 199
199 200 def test_continued_line():
200 201 lines = MULTILINE_MAGIC_ASSIGN[0]
201 202 assert ipt2.find_end_of_continued_line(lines, 1) == 2
202 203
203 204 assert ipt2.assemble_continued_line(lines, (1, 5), 2) == "foo bar"
204 205
205 206
206 207 def test_find_assign_magic():
207 208 check_find(ipt2.MagicAssign, MULTILINE_MAGIC_ASSIGN)
208 209 check_find(ipt2.MagicAssign, MULTILINE_SYSTEM_ASSIGN, match=False)
209 210 check_find(ipt2.MagicAssign, MULTILINE_SYSTEM_ASSIGN_AFTER_DEDENT, match=False)
210 211
211 212
212 213 def test_transform_assign_magic():
213 214 check_transform(ipt2.MagicAssign, MULTILINE_MAGIC_ASSIGN)
214 215
215 216
216 217 def test_find_assign_system():
217 218 check_find(ipt2.SystemAssign, MULTILINE_SYSTEM_ASSIGN)
218 219 check_find(ipt2.SystemAssign, MULTILINE_SYSTEM_ASSIGN_AFTER_DEDENT)
219 220 check_find(ipt2.SystemAssign, (["a = !ls\n"], (1, 5), None))
220 221 check_find(ipt2.SystemAssign, (["a=!ls\n"], (1, 2), None))
221 222 check_find(ipt2.SystemAssign, MULTILINE_MAGIC_ASSIGN, match=False)
222 223
223 224
224 225 def test_transform_assign_system():
225 226 check_transform(ipt2.SystemAssign, MULTILINE_SYSTEM_ASSIGN)
226 227 check_transform(ipt2.SystemAssign, MULTILINE_SYSTEM_ASSIGN_AFTER_DEDENT)
227 228
228 229
229 230 def test_find_magic_escape():
230 231 check_find(ipt2.EscapedCommand, MULTILINE_MAGIC)
231 232 check_find(ipt2.EscapedCommand, INDENTED_MAGIC)
232 233 check_find(ipt2.EscapedCommand, MULTILINE_MAGIC_ASSIGN, match=False)
233 234
234 235
235 236 def test_transform_magic_escape():
236 237 check_transform(ipt2.EscapedCommand, MULTILINE_MAGIC)
237 238 check_transform(ipt2.EscapedCommand, INDENTED_MAGIC)
238 239 check_transform(ipt2.EscapedCommand, CRLF_MAGIC)
239 240
240 241
241 242 def test_find_autocalls():
242 243 for case in [AUTOCALL_QUOTE, AUTOCALL_QUOTE2, AUTOCALL_PAREN]:
243 244 print("Testing %r" % case[0])
244 245 check_find(ipt2.EscapedCommand, case)
245 246
246 247
247 248 def test_transform_autocall():
248 249 for case in [AUTOCALL_QUOTE, AUTOCALL_QUOTE2, AUTOCALL_PAREN]:
249 250 print("Testing %r" % case[0])
250 251 check_transform(ipt2.EscapedCommand, case)
251 252
252 253
253 254 def test_find_help():
254 255 for case in [SIMPLE_HELP, DETAILED_HELP, MAGIC_HELP, HELP_IN_EXPR]:
255 256 check_find(ipt2.HelpEnd, case)
256 257
257 258 tf = check_find(ipt2.HelpEnd, HELP_CONTINUED_LINE)
258 259 assert tf.q_line == 1
259 260 assert tf.q_col == 3
260 261
261 262 tf = check_find(ipt2.HelpEnd, HELP_MULTILINE)
262 263 assert tf.q_line == 1
263 264 assert tf.q_col == 8
264 265
265 266 # ? in a comment does not trigger help
266 267 check_find(ipt2.HelpEnd, (["foo # bar?\n"], None, None), match=False)
267 268 # Nor in a string
268 269 check_find(ipt2.HelpEnd, (["foo = '''bar?\n"], None, None), match=False)
269 270
270 271
271 272 def test_transform_help():
272 273 tf = ipt2.HelpEnd((1, 0), (1, 9))
273 274 assert tf.transform(HELP_IN_EXPR[0]) == HELP_IN_EXPR[2]
274 275
275 276 tf = ipt2.HelpEnd((1, 0), (2, 3))
276 277 assert tf.transform(HELP_CONTINUED_LINE[0]) == HELP_CONTINUED_LINE[2]
277 278
278 279 tf = ipt2.HelpEnd((1, 0), (2, 8))
279 280 assert tf.transform(HELP_MULTILINE[0]) == HELP_MULTILINE[2]
280 281
281 282 tf = ipt2.HelpEnd((1, 0), (1, 0))
282 283 assert tf.transform(HELP_UNICODE[0]) == HELP_UNICODE[2]
283 284
284 285
285 286 def test_find_assign_op_dedent():
286 287 """
287 288 be careful that empty token like dedent are not counted as parens
288 289 """
289 290
290 291 class Tk:
291 292 def __init__(self, s):
292 293 self.string = s
293 294
294 295 assert _find_assign_op([Tk(s) for s in ("", "a", "=", "b")]) == 2
295 296 assert (
296 297 _find_assign_op([Tk(s) for s in ("", "(", "a", "=", "b", ")", "=", "5")]) == 6
297 298 )
298 299
299 300
300 301 examples = [
301 302 pytest.param("a = 1", "complete", None),
302 303 pytest.param("for a in range(5):", "incomplete", 4),
303 304 pytest.param("for a in range(5):\n if a > 0:", "incomplete", 8),
304 305 pytest.param("raise = 2", "invalid", None),
305 306 pytest.param("a = [1,\n2,", "incomplete", 0),
306 307 pytest.param("(\n))", "incomplete", 0),
307 308 pytest.param("\\\r\n", "incomplete", 0),
308 309 pytest.param("a = '''\n hi", "incomplete", 3),
309 310 pytest.param("def a():\n x=1\n global x", "invalid", None),
310 311 pytest.param(
311 312 "a \\ ",
312 313 "invalid",
313 314 None,
314 315 marks=pytest.mark.xfail(
315 316 reason="Bug in python 3.9.8 – bpo 45738",
316 317 condition=sys.version_info
317 318 in [(3, 9, 8, "final", 0), (3, 11, 0, "alpha", 2)],
318 319 raises=SystemError,
319 320 strict=True,
320 321 ),
321 322 ), # Nothing allowed after backslash,
322 323 pytest.param("1\\\n+2", "complete", None),
323 324 ]
324 325
325 326
326 327 @pytest.mark.parametrize("code, expected, number", examples)
327 328 def test_check_complete_param(code, expected, number):
328 329 cc = ipt2.TransformerManager().check_complete
329 330 assert cc(code) == (expected, number)
330 331
331 332
332 333 @pytest.mark.xfail(platform.python_implementation() == "PyPy", reason="fail on pypy")
333 334 @pytest.mark.xfail(
334 335 reason="Bug in python 3.9.8 – bpo 45738",
335 336 condition=sys.version_info in [(3, 9, 8, "final", 0), (3, 11, 0, "alpha", 2)],
336 337 raises=SystemError,
337 338 strict=True,
338 339 )
339 340 def test_check_complete():
340 341 cc = ipt2.TransformerManager().check_complete
341 342
342 343 example = dedent(
343 344 """
344 345 if True:
345 346 a=1"""
346 347 )
347 348
348 349 assert cc(example) == ("incomplete", 4)
349 350 assert cc(example + "\n") == ("complete", None)
350 351 assert cc(example + "\n ") == ("complete", None)
351 352
352 353 # no need to loop on all the letters/numbers.
353 354 short = "12abAB" + string.printable[62:]
354 355 for c in short:
355 356 # test does not raise:
356 357 cc(c)
357 358 for k in short:
358 359 cc(c + k)
359 360
360 361 assert cc("def f():\n x=0\n \\\n ") == ("incomplete", 2)
361 362
362 363
363 364 @pytest.mark.xfail(platform.python_implementation() == "PyPy", reason="fail on pypy")
364 365 @pytest.mark.parametrize(
365 366 "value, expected",
366 367 [
367 368 ('''def foo():\n """''', ("incomplete", 4)),
368 369 ("""async with example:\n pass""", ("incomplete", 4)),
369 370 ("""async with example:\n pass\n """, ("complete", None)),
370 371 ],
371 372 )
372 373 def test_check_complete_II(value, expected):
373 374 """
374 375 Test that multiple line strings are properly handled.
375 376
376 377 Separate test function for convenience
377 378
378 379 """
379 380 cc = ipt2.TransformerManager().check_complete
380 381 assert cc(value) == expected
381 382
382 383
383 384 @pytest.mark.parametrize(
384 385 "value, expected",
385 386 [
386 387 (")", ("invalid", None)),
387 388 ("]", ("invalid", None)),
388 389 ("}", ("invalid", None)),
389 390 (")(", ("invalid", None)),
390 391 ("][", ("invalid", None)),
391 392 ("}{", ("invalid", None)),
392 393 ("]()(", ("invalid", None)),
393 394 ("())(", ("invalid", None)),
394 395 (")[](", ("invalid", None)),
395 396 ("()](", ("invalid", None)),
396 397 ],
397 398 )
398 399 def test_check_complete_invalidates_sunken_brackets(value, expected):
399 400 """
400 401 Test that a single line with more closing brackets than the opening ones is
401 402 interpreted as invalid
402 403 """
403 404 cc = ipt2.TransformerManager().check_complete
404 405 assert cc(value) == expected
405 406
406 407
407 408 def test_null_cleanup_transformer():
408 409 manager = ipt2.TransformerManager()
409 410 manager.cleanup_transforms.insert(0, null_cleanup_transformer)
410 411 assert manager.transform_cell("") == ""
411 412
412 413
413 414 def test_side_effects_I():
414 415 count = 0
415 416
416 417 def counter(lines):
417 418 nonlocal count
418 419 count += 1
419 420 return lines
420 421
421 422 counter.has_side_effects = True
422 423
423 424 manager = ipt2.TransformerManager()
424 425 manager.cleanup_transforms.insert(0, counter)
425 426 assert manager.check_complete("a=1\n") == ("complete", None)
426 427 assert count == 0
427 428
428 429
429 430 def test_side_effects_II():
430 431 count = 0
431 432
432 433 def counter(lines):
433 434 nonlocal count
434 435 count += 1
435 436 return lines
436 437
437 438 counter.has_side_effects = True
438 439
439 440 manager = ipt2.TransformerManager()
440 441 manager.line_transforms.insert(0, counter)
441 442 assert manager.check_complete("b=1\n") == ("complete", None)
442 443 assert count == 0
@@ -1,1100 +1,1100 b''
1 1 # -*- coding: utf-8 -*-
2 2 """Tests for the key interactiveshell module.
3 3
4 4 Historically the main classes in interactiveshell have been under-tested. This
5 5 module should grow as many single-method tests as possible to trap many of the
6 6 recurring bugs we seem to encounter with high-level interaction.
7 7 """
8 8
9 9 # Copyright (c) IPython Development Team.
10 10 # Distributed under the terms of the Modified BSD License.
11 11
12 12 import asyncio
13 13 import ast
14 14 import os
15 15 import signal
16 16 import shutil
17 17 import sys
18 18 import tempfile
19 19 import unittest
20 20 from unittest import mock
21 21
22 22 from os.path import join
23 23
24 24 from IPython.core.error import InputRejected
25 25 from IPython.core.inputtransformer import InputTransformer
26 26 from IPython.core import interactiveshell
27 27 from IPython.testing.decorators import (
28 28 skipif, skip_win32, onlyif_unicode_paths, onlyif_cmds_exist,
29 29 )
30 30 from IPython.testing import tools as tt
31 31 from IPython.utils.process import find_cmd
32 32
33 33 #-----------------------------------------------------------------------------
34 34 # Globals
35 35 #-----------------------------------------------------------------------------
36 36 # This is used by every single test, no point repeating it ad nauseam
37 37
38 38 #-----------------------------------------------------------------------------
39 39 # Tests
40 40 #-----------------------------------------------------------------------------
41 41
42 42 class DerivedInterrupt(KeyboardInterrupt):
43 43 pass
44 44
45 45 class InteractiveShellTestCase(unittest.TestCase):
46 46 def test_naked_string_cells(self):
47 47 """Test that cells with only naked strings are fully executed"""
48 48 # First, single-line inputs
49 49 ip.run_cell('"a"\n')
50 50 self.assertEqual(ip.user_ns['_'], 'a')
51 51 # And also multi-line cells
52 52 ip.run_cell('"""a\nb"""\n')
53 53 self.assertEqual(ip.user_ns['_'], 'a\nb')
54 54
55 55 def test_run_empty_cell(self):
56 56 """Just make sure we don't get a horrible error with a blank
57 57 cell of input. Yes, I did overlook that."""
58 58 old_xc = ip.execution_count
59 59 res = ip.run_cell('')
60 60 self.assertEqual(ip.execution_count, old_xc)
61 61 self.assertEqual(res.execution_count, None)
62 62
63 63 def test_run_cell_multiline(self):
64 64 """Multi-block, multi-line cells must execute correctly.
65 65 """
66 66 src = '\n'.join(["x=1",
67 67 "y=2",
68 68 "if 1:",
69 69 " x += 1",
70 70 " y += 1",])
71 71 res = ip.run_cell(src)
72 72 self.assertEqual(ip.user_ns['x'], 2)
73 73 self.assertEqual(ip.user_ns['y'], 3)
74 74 self.assertEqual(res.success, True)
75 75 self.assertEqual(res.result, None)
76 76
77 77 def test_multiline_string_cells(self):
78 78 "Code sprinkled with multiline strings should execute (GH-306)"
79 79 ip.run_cell('tmp=0')
80 80 self.assertEqual(ip.user_ns['tmp'], 0)
81 81 res = ip.run_cell('tmp=1;"""a\nb"""\n')
82 82 self.assertEqual(ip.user_ns['tmp'], 1)
83 83 self.assertEqual(res.success, True)
84 84 self.assertEqual(res.result, "a\nb")
85 85
86 86 def test_dont_cache_with_semicolon(self):
87 87 "Ending a line with semicolon should not cache the returned object (GH-307)"
88 88 oldlen = len(ip.user_ns['Out'])
89 89 for cell in ['1;', '1;1;']:
90 90 res = ip.run_cell(cell, store_history=True)
91 91 newlen = len(ip.user_ns['Out'])
92 92 self.assertEqual(oldlen, newlen)
93 93 self.assertIsNone(res.result)
94 94 i = 0
95 95 #also test the default caching behavior
96 96 for cell in ['1', '1;1']:
97 97 ip.run_cell(cell, store_history=True)
98 98 newlen = len(ip.user_ns['Out'])
99 99 i += 1
100 100 self.assertEqual(oldlen+i, newlen)
101 101
102 102 def test_syntax_error(self):
103 103 res = ip.run_cell("raise = 3")
104 104 self.assertIsInstance(res.error_before_exec, SyntaxError)
105 105
106 106 def test_In_variable(self):
107 107 "Verify that In variable grows with user input (GH-284)"
108 108 oldlen = len(ip.user_ns['In'])
109 109 ip.run_cell('1;', store_history=True)
110 110 newlen = len(ip.user_ns['In'])
111 111 self.assertEqual(oldlen+1, newlen)
112 112 self.assertEqual(ip.user_ns['In'][-1],'1;')
113 113
114 114 def test_magic_names_in_string(self):
115 115 ip.run_cell('a = """\n%exit\n"""')
116 116 self.assertEqual(ip.user_ns['a'], '\n%exit\n')
117 117
118 118 def test_trailing_newline(self):
119 119 """test that running !(command) does not raise a SyntaxError"""
120 120 ip.run_cell('!(true)\n', False)
121 121 ip.run_cell('!(true)\n\n\n', False)
122 122
123 123 def test_gh_597(self):
124 124 """Pretty-printing lists of objects with non-ascii reprs may cause
125 125 problems."""
126 126 class Spam(object):
127 127 def __repr__(self):
128 128 return "\xe9"*50
129 129 import IPython.core.formatters
130 130 f = IPython.core.formatters.PlainTextFormatter()
131 131 f([Spam(),Spam()])
132 132
133 133
134 134 def test_future_flags(self):
135 135 """Check that future flags are used for parsing code (gh-777)"""
136 136 ip.run_cell('from __future__ import barry_as_FLUFL')
137 137 try:
138 138 ip.run_cell('prfunc_return_val = 1 <> 2')
139 139 assert 'prfunc_return_val' in ip.user_ns
140 140 finally:
141 141 # Reset compiler flags so we don't mess up other tests.
142 142 ip.compile.reset_compiler_flags()
143 143
144 144 def test_can_pickle(self):
145 145 "Can we pickle objects defined interactively (GH-29)"
146 146 ip = get_ipython()
147 147 ip.reset()
148 148 ip.run_cell(("class Mylist(list):\n"
149 149 " def __init__(self,x=[]):\n"
150 150 " list.__init__(self,x)"))
151 151 ip.run_cell("w=Mylist([1,2,3])")
152 152
153 153 from pickle import dumps
154 154
155 155 # We need to swap in our main module - this is only necessary
156 156 # inside the test framework, because IPython puts the interactive module
157 157 # in place (but the test framework undoes this).
158 158 _main = sys.modules['__main__']
159 159 sys.modules['__main__'] = ip.user_module
160 160 try:
161 161 res = dumps(ip.user_ns["w"])
162 162 finally:
163 163 sys.modules['__main__'] = _main
164 164 self.assertTrue(isinstance(res, bytes))
165 165
166 166 def test_global_ns(self):
167 167 "Code in functions must be able to access variables outside them."
168 168 ip = get_ipython()
169 169 ip.run_cell("a = 10")
170 170 ip.run_cell(("def f(x):\n"
171 171 " return x + a"))
172 172 ip.run_cell("b = f(12)")
173 173 self.assertEqual(ip.user_ns["b"], 22)
174 174
175 175 def test_bad_custom_tb(self):
176 176 """Check that InteractiveShell is protected from bad custom exception handlers"""
177 177 ip.set_custom_exc((IOError,), lambda etype,value,tb: 1/0)
178 178 self.assertEqual(ip.custom_exceptions, (IOError,))
179 179 with tt.AssertPrints("Custom TB Handler failed", channel='stderr'):
180 180 ip.run_cell(u'raise IOError("foo")')
181 181 self.assertEqual(ip.custom_exceptions, ())
182 182
183 183 def test_bad_custom_tb_return(self):
184 184 """Check that InteractiveShell is protected from bad return types in custom exception handlers"""
185 185 ip.set_custom_exc((NameError,),lambda etype,value,tb, tb_offset=None: 1)
186 186 self.assertEqual(ip.custom_exceptions, (NameError,))
187 187 with tt.AssertPrints("Custom TB Handler failed", channel='stderr'):
188 188 ip.run_cell(u'a=abracadabra')
189 189 self.assertEqual(ip.custom_exceptions, ())
190 190
191 191 def test_drop_by_id(self):
192 192 myvars = {"a":object(), "b":object(), "c": object()}
193 193 ip.push(myvars, interactive=False)
194 194 for name in myvars:
195 195 assert name in ip.user_ns, name
196 196 assert name in ip.user_ns_hidden, name
197 197 ip.user_ns['b'] = 12
198 198 ip.drop_by_id(myvars)
199 199 for name in ["a", "c"]:
200 200 assert name not in ip.user_ns, name
201 201 assert name not in ip.user_ns_hidden, name
202 202 assert ip.user_ns['b'] == 12
203 203 ip.reset()
204 204
205 205 def test_var_expand(self):
206 206 ip.user_ns['f'] = u'Ca\xf1o'
207 207 self.assertEqual(ip.var_expand(u'echo $f'), u'echo Ca\xf1o')
208 208 self.assertEqual(ip.var_expand(u'echo {f}'), u'echo Ca\xf1o')
209 209 self.assertEqual(ip.var_expand(u'echo {f[:-1]}'), u'echo Ca\xf1')
210 210 self.assertEqual(ip.var_expand(u'echo {1*2}'), u'echo 2')
211 211
212 212 self.assertEqual(ip.var_expand(u"grep x | awk '{print $1}'"), u"grep x | awk '{print $1}'")
213 213
214 214 ip.user_ns['f'] = b'Ca\xc3\xb1o'
215 215 # This should not raise any exception:
216 216 ip.var_expand(u'echo $f')
217 217
218 218 def test_var_expand_local(self):
219 219 """Test local variable expansion in !system and %magic calls"""
220 220 # !system
221 221 ip.run_cell(
222 222 "def test():\n"
223 223 ' lvar = "ttt"\n'
224 224 " ret = !echo {lvar}\n"
225 225 " return ret[0]\n"
226 226 )
227 227 res = ip.user_ns["test"]()
228 228 self.assertIn("ttt", res)
229 229
230 230 # %magic
231 231 ip.run_cell(
232 232 "def makemacro():\n"
233 233 ' macroname = "macro_var_expand_locals"\n'
234 234 " %macro {macroname} codestr\n"
235 235 )
236 236 ip.user_ns["codestr"] = "str(12)"
237 237 ip.run_cell("makemacro()")
238 238 self.assertIn("macro_var_expand_locals", ip.user_ns)
239 239
240 240 def test_var_expand_self(self):
241 241 """Test variable expansion with the name 'self', which was failing.
242 242
243 243 See https://github.com/ipython/ipython/issues/1878#issuecomment-7698218
244 244 """
245 245 ip.run_cell(
246 246 "class cTest:\n"
247 247 ' classvar="see me"\n'
248 248 " def test(self):\n"
249 249 " res = !echo Variable: {self.classvar}\n"
250 250 " return res[0]\n"
251 251 )
252 252 self.assertIn("see me", ip.user_ns["cTest"]().test())
253 253
254 254 def test_bad_var_expand(self):
255 255 """var_expand on invalid formats shouldn't raise"""
256 256 # SyntaxError
257 257 self.assertEqual(ip.var_expand(u"{'a':5}"), u"{'a':5}")
258 258 # NameError
259 259 self.assertEqual(ip.var_expand(u"{asdf}"), u"{asdf}")
260 260 # ZeroDivisionError
261 261 self.assertEqual(ip.var_expand(u"{1/0}"), u"{1/0}")
262 262
263 263 def test_silent_postexec(self):
264 264 """run_cell(silent=True) doesn't invoke pre/post_run_cell callbacks"""
265 265 pre_explicit = mock.Mock()
266 266 pre_always = mock.Mock()
267 267 post_explicit = mock.Mock()
268 268 post_always = mock.Mock()
269 269 all_mocks = [pre_explicit, pre_always, post_explicit, post_always]
270 270
271 271 ip.events.register('pre_run_cell', pre_explicit)
272 272 ip.events.register('pre_execute', pre_always)
273 273 ip.events.register('post_run_cell', post_explicit)
274 274 ip.events.register('post_execute', post_always)
275 275
276 276 try:
277 277 ip.run_cell("1", silent=True)
278 278 assert pre_always.called
279 279 assert not pre_explicit.called
280 280 assert post_always.called
281 281 assert not post_explicit.called
282 282 # double-check that non-silent exec did what we expected
283 283 # silent to avoid
284 284 ip.run_cell("1")
285 285 assert pre_explicit.called
286 286 assert post_explicit.called
287 287 info, = pre_explicit.call_args[0]
288 288 result, = post_explicit.call_args[0]
289 289 self.assertEqual(info, result.info)
290 290 # check that post hooks are always called
291 291 [m.reset_mock() for m in all_mocks]
292 292 ip.run_cell("syntax error")
293 293 assert pre_always.called
294 294 assert pre_explicit.called
295 295 assert post_always.called
296 296 assert post_explicit.called
297 297 info, = pre_explicit.call_args[0]
298 298 result, = post_explicit.call_args[0]
299 299 self.assertEqual(info, result.info)
300 300 finally:
301 301 # remove post-exec
302 302 ip.events.unregister('pre_run_cell', pre_explicit)
303 303 ip.events.unregister('pre_execute', pre_always)
304 304 ip.events.unregister('post_run_cell', post_explicit)
305 305 ip.events.unregister('post_execute', post_always)
306 306
307 307 def test_silent_noadvance(self):
308 308 """run_cell(silent=True) doesn't advance execution_count"""
309 309 ec = ip.execution_count
310 310 # silent should force store_history=False
311 311 ip.run_cell("1", store_history=True, silent=True)
312 312
313 313 self.assertEqual(ec, ip.execution_count)
314 314 # double-check that non-silent exec did what we expected
315 315 # silent to avoid
316 316 ip.run_cell("1", store_history=True)
317 317 self.assertEqual(ec+1, ip.execution_count)
318 318
319 319 def test_silent_nodisplayhook(self):
320 320 """run_cell(silent=True) doesn't trigger displayhook"""
321 321 d = dict(called=False)
322 322
323 323 trap = ip.display_trap
324 324 save_hook = trap.hook
325 325
326 326 def failing_hook(*args, **kwargs):
327 327 d['called'] = True
328 328
329 329 try:
330 330 trap.hook = failing_hook
331 331 res = ip.run_cell("1", silent=True)
332 332 self.assertFalse(d['called'])
333 333 self.assertIsNone(res.result)
334 334 # double-check that non-silent exec did what we expected
335 335 # silent to avoid
336 336 ip.run_cell("1")
337 337 self.assertTrue(d['called'])
338 338 finally:
339 339 trap.hook = save_hook
340 340
341 341 def test_ofind_line_magic(self):
342 342 from IPython.core.magic import register_line_magic
343 343
344 344 @register_line_magic
345 345 def lmagic(line):
346 346 "A line magic"
347 347
348 348 # Get info on line magic
349 349 lfind = ip._ofind("lmagic")
350 350 info = dict(
351 351 found=True,
352 352 isalias=False,
353 353 ismagic=True,
354 354 namespace="IPython internal",
355 355 obj=lmagic,
356 356 parent=None,
357 357 )
358 358 self.assertEqual(lfind, info)
359 359
360 360 def test_ofind_cell_magic(self):
361 361 from IPython.core.magic import register_cell_magic
362 362
363 363 @register_cell_magic
364 364 def cmagic(line, cell):
365 365 "A cell magic"
366 366
367 367 # Get info on cell magic
368 368 find = ip._ofind("cmagic")
369 369 info = dict(
370 370 found=True,
371 371 isalias=False,
372 372 ismagic=True,
373 373 namespace="IPython internal",
374 374 obj=cmagic,
375 375 parent=None,
376 376 )
377 377 self.assertEqual(find, info)
378 378
379 379 def test_ofind_property_with_error(self):
380 380 class A(object):
381 381 @property
382 382 def foo(self):
383 383 raise NotImplementedError() # pragma: no cover
384 384
385 385 a = A()
386 386
387 387 found = ip._ofind('a.foo', [('locals', locals())])
388 388 info = dict(found=True, isalias=False, ismagic=False,
389 389 namespace='locals', obj=A.foo, parent=a)
390 390 self.assertEqual(found, info)
391 391
392 392 def test_ofind_multiple_attribute_lookups(self):
393 393 class A(object):
394 394 @property
395 395 def foo(self):
396 396 raise NotImplementedError() # pragma: no cover
397 397
398 398 a = A()
399 399 a.a = A()
400 400 a.a.a = A()
401 401
402 402 found = ip._ofind('a.a.a.foo', [('locals', locals())])
403 403 info = dict(found=True, isalias=False, ismagic=False,
404 404 namespace='locals', obj=A.foo, parent=a.a.a)
405 405 self.assertEqual(found, info)
406 406
407 407 def test_ofind_slotted_attributes(self):
408 408 class A(object):
409 409 __slots__ = ['foo']
410 410 def __init__(self):
411 411 self.foo = 'bar'
412 412
413 413 a = A()
414 414 found = ip._ofind('a.foo', [('locals', locals())])
415 415 info = dict(found=True, isalias=False, ismagic=False,
416 416 namespace='locals', obj=a.foo, parent=a)
417 417 self.assertEqual(found, info)
418 418
419 419 found = ip._ofind('a.bar', [('locals', locals())])
420 420 info = dict(found=False, isalias=False, ismagic=False,
421 421 namespace=None, obj=None, parent=a)
422 422 self.assertEqual(found, info)
423 423
424 424 def test_ofind_prefers_property_to_instance_level_attribute(self):
425 425 class A(object):
426 426 @property
427 427 def foo(self):
428 428 return 'bar'
429 429 a = A()
430 430 a.__dict__["foo"] = "baz"
431 431 self.assertEqual(a.foo, "bar")
432 432 found = ip._ofind("a.foo", [("locals", locals())])
433 433 self.assertIs(found["obj"], A.foo)
434 434
435 435 def test_custom_syntaxerror_exception(self):
436 436 called = []
437 437 def my_handler(shell, etype, value, tb, tb_offset=None):
438 438 called.append(etype)
439 439 shell.showtraceback((etype, value, tb), tb_offset=tb_offset)
440 440
441 441 ip.set_custom_exc((SyntaxError,), my_handler)
442 442 try:
443 443 ip.run_cell("1f")
444 444 # Check that this was called, and only once.
445 445 self.assertEqual(called, [SyntaxError])
446 446 finally:
447 447 # Reset the custom exception hook
448 448 ip.set_custom_exc((), None)
449 449
450 450 def test_custom_exception(self):
451 451 called = []
452 452 def my_handler(shell, etype, value, tb, tb_offset=None):
453 453 called.append(etype)
454 454 shell.showtraceback((etype, value, tb), tb_offset=tb_offset)
455 455
456 456 ip.set_custom_exc((ValueError,), my_handler)
457 457 try:
458 458 res = ip.run_cell("raise ValueError('test')")
459 459 # Check that this was called, and only once.
460 460 self.assertEqual(called, [ValueError])
461 461 # Check that the error is on the result object
462 462 self.assertIsInstance(res.error_in_exec, ValueError)
463 463 finally:
464 464 # Reset the custom exception hook
465 465 ip.set_custom_exc((), None)
466 466
467 467 @mock.patch("builtins.print")
468 468 def test_showtraceback_with_surrogates(self, mocked_print):
469 469 values = []
470 470
471 471 def mock_print_func(value, sep=" ", end="\n", file=sys.stdout, flush=False):
472 472 values.append(value)
473 473 if value == chr(0xD8FF):
474 474 raise UnicodeEncodeError("utf-8", chr(0xD8FF), 0, 1, "")
475 475
476 476 # mock builtins.print
477 477 mocked_print.side_effect = mock_print_func
478 478
479 479 # ip._showtraceback() is replaced in globalipapp.py.
480 480 # Call original method to test.
481 481 interactiveshell.InteractiveShell._showtraceback(ip, None, None, chr(0xD8FF))
482 482
483 483 self.assertEqual(mocked_print.call_count, 2)
484 484 self.assertEqual(values, [chr(0xD8FF), "\\ud8ff"])
485 485
486 486 def test_mktempfile(self):
487 487 filename = ip.mktempfile()
488 488 # Check that we can open the file again on Windows
489 489 with open(filename, "w", encoding="utf-8") as f:
490 490 f.write("abc")
491 491
492 492 filename = ip.mktempfile(data="blah")
493 493 with open(filename, "r", encoding="utf-8") as f:
494 494 self.assertEqual(f.read(), "blah")
495 495
496 496 def test_new_main_mod(self):
497 497 # Smoketest to check that this accepts a unicode module name
498 498 name = u'jiefmw'
499 499 mod = ip.new_main_mod(u'%s.py' % name, name)
500 500 self.assertEqual(mod.__name__, name)
501 501
502 502 def test_get_exception_only(self):
503 503 try:
504 504 raise KeyboardInterrupt
505 505 except KeyboardInterrupt:
506 506 msg = ip.get_exception_only()
507 507 self.assertEqual(msg, 'KeyboardInterrupt\n')
508 508
509 509 try:
510 510 raise DerivedInterrupt("foo")
511 511 except KeyboardInterrupt:
512 512 msg = ip.get_exception_only()
513 513 self.assertEqual(msg, 'IPython.core.tests.test_interactiveshell.DerivedInterrupt: foo\n')
514 514
515 515 def test_inspect_text(self):
516 516 ip.run_cell('a = 5')
517 517 text = ip.object_inspect_text('a')
518 518 self.assertIsInstance(text, str)
519 519
520 520 def test_last_execution_result(self):
521 521 """ Check that last execution result gets set correctly (GH-10702) """
522 522 result = ip.run_cell('a = 5; a')
523 523 self.assertTrue(ip.last_execution_succeeded)
524 524 self.assertEqual(ip.last_execution_result.result, 5)
525 525
526 526 result = ip.run_cell('a = x_invalid_id_x')
527 527 self.assertFalse(ip.last_execution_succeeded)
528 528 self.assertFalse(ip.last_execution_result.success)
529 529 self.assertIsInstance(ip.last_execution_result.error_in_exec, NameError)
530 530
531 531 def test_reset_aliasing(self):
532 532 """ Check that standard posix aliases work after %reset. """
533 533 if os.name != 'posix':
534 534 return
535 535
536 536 ip.reset()
537 537 for cmd in ('clear', 'more', 'less', 'man'):
538 538 res = ip.run_cell('%' + cmd)
539 539 self.assertEqual(res.success, True)
540 540
541 541
542 542 class TestSafeExecfileNonAsciiPath(unittest.TestCase):
543 543
544 544 @onlyif_unicode_paths
545 545 def setUp(self):
546 546 self.BASETESTDIR = tempfile.mkdtemp()
547 547 self.TESTDIR = join(self.BASETESTDIR, u"åäö")
548 548 os.mkdir(self.TESTDIR)
549 549 with open(
550 join(self.TESTDIR, u"åäötestscript.py"), "w", encoding="utf-8"
550 join(self.TESTDIR, "åäötestscript.py"), "w", encoding="utf-8"
551 551 ) as sfile:
552 552 sfile.write("pass\n")
553 553 self.oldpath = os.getcwd()
554 554 os.chdir(self.TESTDIR)
555 555 self.fname = u"åäötestscript.py"
556 556
557 557 def tearDown(self):
558 558 os.chdir(self.oldpath)
559 559 shutil.rmtree(self.BASETESTDIR)
560 560
561 561 @onlyif_unicode_paths
562 562 def test_1(self):
563 563 """Test safe_execfile with non-ascii path
564 564 """
565 565 ip.safe_execfile(self.fname, {}, raise_exceptions=True)
566 566
567 567 class ExitCodeChecks(tt.TempFileMixin):
568 568
569 569 def setUp(self):
570 570 self.system = ip.system_raw
571 571
572 572 def test_exit_code_ok(self):
573 573 self.system('exit 0')
574 574 self.assertEqual(ip.user_ns['_exit_code'], 0)
575 575
576 576 def test_exit_code_error(self):
577 577 self.system('exit 1')
578 578 self.assertEqual(ip.user_ns['_exit_code'], 1)
579 579
580 580 @skipif(not hasattr(signal, 'SIGALRM'))
581 581 def test_exit_code_signal(self):
582 582 self.mktmp("import signal, time\n"
583 583 "signal.setitimer(signal.ITIMER_REAL, 0.1)\n"
584 584 "time.sleep(1)\n")
585 585 self.system("%s %s" % (sys.executable, self.fname))
586 586 self.assertEqual(ip.user_ns['_exit_code'], -signal.SIGALRM)
587 587
588 588 @onlyif_cmds_exist("csh")
589 589 def test_exit_code_signal_csh(self): # pragma: no cover
590 590 SHELL = os.environ.get("SHELL", None)
591 591 os.environ["SHELL"] = find_cmd("csh")
592 592 try:
593 593 self.test_exit_code_signal()
594 594 finally:
595 595 if SHELL is not None:
596 596 os.environ['SHELL'] = SHELL
597 597 else:
598 598 del os.environ['SHELL']
599 599
600 600
601 601 class TestSystemRaw(ExitCodeChecks):
602 602
603 603 def setUp(self):
604 604 super().setUp()
605 605 self.system = ip.system_raw
606 606
607 607 @onlyif_unicode_paths
608 608 def test_1(self):
609 609 """Test system_raw with non-ascii cmd
610 610 """
611 611 cmd = u'''python -c "'åäö'" '''
612 612 ip.system_raw(cmd)
613 613
614 614 @mock.patch('subprocess.call', side_effect=KeyboardInterrupt)
615 615 @mock.patch('os.system', side_effect=KeyboardInterrupt)
616 616 def test_control_c(self, *mocks):
617 617 try:
618 618 self.system("sleep 1 # wont happen")
619 619 except KeyboardInterrupt: # pragma: no cove
620 620 self.fail(
621 621 "system call should intercept "
622 622 "keyboard interrupt from subprocess.call"
623 623 )
624 624 self.assertEqual(ip.user_ns["_exit_code"], -signal.SIGINT)
625 625
626 626 def test_magic_warnings(self):
627 627 for magic_cmd in ("pip", "conda", "cd"):
628 628 with self.assertWarnsRegex(Warning, "You executed the system command"):
629 629 ip.system_raw(magic_cmd)
630 630
631 631 # TODO: Exit codes are currently ignored on Windows.
632 632 class TestSystemPipedExitCode(ExitCodeChecks):
633 633
634 634 def setUp(self):
635 635 super().setUp()
636 636 self.system = ip.system_piped
637 637
638 638 @skip_win32
639 639 def test_exit_code_ok(self):
640 640 ExitCodeChecks.test_exit_code_ok(self)
641 641
642 642 @skip_win32
643 643 def test_exit_code_error(self):
644 644 ExitCodeChecks.test_exit_code_error(self)
645 645
646 646 @skip_win32
647 647 def test_exit_code_signal(self):
648 648 ExitCodeChecks.test_exit_code_signal(self)
649 649
650 650 class TestModules(tt.TempFileMixin):
651 651 def test_extraneous_loads(self):
652 652 """Test we're not loading modules on startup that we shouldn't.
653 653 """
654 654 self.mktmp("import sys\n"
655 655 "print('numpy' in sys.modules)\n"
656 656 "print('ipyparallel' in sys.modules)\n"
657 657 "print('ipykernel' in sys.modules)\n"
658 658 )
659 659 out = "False\nFalse\nFalse\n"
660 660 tt.ipexec_validate(self.fname, out)
661 661
662 662 class Negator(ast.NodeTransformer):
663 663 """Negates all number literals in an AST."""
664 664
665 665 # for python 3.7 and earlier
666 666 def visit_Num(self, node):
667 667 node.n = -node.n
668 668 return node
669 669
670 670 # for python 3.8+
671 671 def visit_Constant(self, node):
672 672 if isinstance(node.value, int):
673 673 return self.visit_Num(node)
674 674 return node
675 675
676 676 class TestAstTransform(unittest.TestCase):
677 677 def setUp(self):
678 678 self.negator = Negator()
679 679 ip.ast_transformers.append(self.negator)
680 680
681 681 def tearDown(self):
682 682 ip.ast_transformers.remove(self.negator)
683 683
684 684 def test_non_int_const(self):
685 685 with tt.AssertPrints("hello"):
686 686 ip.run_cell('print("hello")')
687 687
688 688 def test_run_cell(self):
689 689 with tt.AssertPrints("-34"):
690 690 ip.run_cell("print(12 + 22)")
691 691
692 692 # A named reference to a number shouldn't be transformed.
693 693 ip.user_ns["n"] = 55
694 694 with tt.AssertNotPrints("-55"):
695 695 ip.run_cell("print(n)")
696 696
697 697 def test_timeit(self):
698 698 called = set()
699 699 def f(x):
700 700 called.add(x)
701 701 ip.push({'f':f})
702 702
703 703 with tt.AssertPrints("std. dev. of"):
704 704 ip.run_line_magic("timeit", "-n1 f(1)")
705 705 self.assertEqual(called, {-1})
706 706 called.clear()
707 707
708 708 with tt.AssertPrints("std. dev. of"):
709 709 ip.run_cell_magic("timeit", "-n1 f(2)", "f(3)")
710 710 self.assertEqual(called, {-2, -3})
711 711
712 712 def test_time(self):
713 713 called = []
714 714 def f(x):
715 715 called.append(x)
716 716 ip.push({'f':f})
717 717
718 718 # Test with an expression
719 719 with tt.AssertPrints("Wall time: "):
720 720 ip.run_line_magic("time", "f(5+9)")
721 721 self.assertEqual(called, [-14])
722 722 called[:] = []
723 723
724 724 # Test with a statement (different code path)
725 725 with tt.AssertPrints("Wall time: "):
726 726 ip.run_line_magic("time", "a = f(-3 + -2)")
727 727 self.assertEqual(called, [5])
728 728
729 729 def test_macro(self):
730 730 ip.push({'a':10})
731 731 # The AST transformation makes this do a+=-1
732 732 ip.define_macro("amacro", "a+=1\nprint(a)")
733 733
734 734 with tt.AssertPrints("9"):
735 735 ip.run_cell("amacro")
736 736 with tt.AssertPrints("8"):
737 737 ip.run_cell("amacro")
738 738
739 739 class TestMiscTransform(unittest.TestCase):
740 740
741 741
742 742 def test_transform_only_once(self):
743 743 cleanup = 0
744 744 line_t = 0
745 745 def count_cleanup(lines):
746 746 nonlocal cleanup
747 747 cleanup += 1
748 748 return lines
749 749
750 750 def count_line_t(lines):
751 751 nonlocal line_t
752 752 line_t += 1
753 753 return lines
754 754
755 755 ip.input_transformer_manager.cleanup_transforms.append(count_cleanup)
756 756 ip.input_transformer_manager.line_transforms.append(count_line_t)
757 757
758 758 ip.run_cell('1')
759 759
760 760 assert cleanup == 1
761 761 assert line_t == 1
762 762
763 763 class IntegerWrapper(ast.NodeTransformer):
764 764 """Wraps all integers in a call to Integer()"""
765 765
766 766 # for Python 3.7 and earlier
767 767
768 768 # for Python 3.7 and earlier
769 769 def visit_Num(self, node):
770 770 if isinstance(node.n, int):
771 771 return ast.Call(func=ast.Name(id='Integer', ctx=ast.Load()),
772 772 args=[node], keywords=[])
773 773 return node
774 774
775 775 # For Python 3.8+
776 776 def visit_Constant(self, node):
777 777 if isinstance(node.value, int):
778 778 return self.visit_Num(node)
779 779 return node
780 780
781 781
782 782 class TestAstTransform2(unittest.TestCase):
783 783 def setUp(self):
784 784 self.intwrapper = IntegerWrapper()
785 785 ip.ast_transformers.append(self.intwrapper)
786 786
787 787 self.calls = []
788 788 def Integer(*args):
789 789 self.calls.append(args)
790 790 return args
791 791 ip.push({"Integer": Integer})
792 792
793 793 def tearDown(self):
794 794 ip.ast_transformers.remove(self.intwrapper)
795 795 del ip.user_ns['Integer']
796 796
797 797 def test_run_cell(self):
798 798 ip.run_cell("n = 2")
799 799 self.assertEqual(self.calls, [(2,)])
800 800
801 801 # This shouldn't throw an error
802 802 ip.run_cell("o = 2.0")
803 803 self.assertEqual(ip.user_ns['o'], 2.0)
804 804
805 805 def test_run_cell_non_int(self):
806 806 ip.run_cell("n = 'a'")
807 807 assert self.calls == []
808 808
809 809 def test_timeit(self):
810 810 called = set()
811 811 def f(x):
812 812 called.add(x)
813 813 ip.push({'f':f})
814 814
815 815 with tt.AssertPrints("std. dev. of"):
816 816 ip.run_line_magic("timeit", "-n1 f(1)")
817 817 self.assertEqual(called, {(1,)})
818 818 called.clear()
819 819
820 820 with tt.AssertPrints("std. dev. of"):
821 821 ip.run_cell_magic("timeit", "-n1 f(2)", "f(3)")
822 822 self.assertEqual(called, {(2,), (3,)})
823 823
824 824 class ErrorTransformer(ast.NodeTransformer):
825 825 """Throws an error when it sees a number."""
826 826
827 827 def visit_Constant(self, node):
828 828 if isinstance(node.value, int):
829 829 raise ValueError("test")
830 830 return node
831 831
832 832
833 833 class TestAstTransformError(unittest.TestCase):
834 834 def test_unregistering(self):
835 835 err_transformer = ErrorTransformer()
836 836 ip.ast_transformers.append(err_transformer)
837 837
838 838 with self.assertWarnsRegex(UserWarning, "It will be unregistered"):
839 839 ip.run_cell("1 + 2")
840 840
841 841 # This should have been removed.
842 842 self.assertNotIn(err_transformer, ip.ast_transformers)
843 843
844 844
845 845 class StringRejector(ast.NodeTransformer):
846 846 """Throws an InputRejected when it sees a string literal.
847 847
848 848 Used to verify that NodeTransformers can signal that a piece of code should
849 849 not be executed by throwing an InputRejected.
850 850 """
851 851
852 852 # 3.8 only
853 853 def visit_Constant(self, node):
854 854 if isinstance(node.value, str):
855 855 raise InputRejected("test")
856 856 return node
857 857
858 858
859 859 class TestAstTransformInputRejection(unittest.TestCase):
860 860
861 861 def setUp(self):
862 862 self.transformer = StringRejector()
863 863 ip.ast_transformers.append(self.transformer)
864 864
865 865 def tearDown(self):
866 866 ip.ast_transformers.remove(self.transformer)
867 867
868 868 def test_input_rejection(self):
869 869 """Check that NodeTransformers can reject input."""
870 870
871 871 expect_exception_tb = tt.AssertPrints("InputRejected: test")
872 872 expect_no_cell_output = tt.AssertNotPrints("'unsafe'", suppress=False)
873 873
874 874 # Run the same check twice to verify that the transformer is not
875 875 # disabled after raising.
876 876 with expect_exception_tb, expect_no_cell_output:
877 877 ip.run_cell("'unsafe'")
878 878
879 879 with expect_exception_tb, expect_no_cell_output:
880 880 res = ip.run_cell("'unsafe'")
881 881
882 882 self.assertIsInstance(res.error_before_exec, InputRejected)
883 883
884 884 def test__IPYTHON__():
885 885 # This shouldn't raise a NameError, that's all
886 886 __IPYTHON__
887 887
888 888
889 889 class DummyRepr(object):
890 890 def __repr__(self):
891 891 return "DummyRepr"
892 892
893 893 def _repr_html_(self):
894 894 return "<b>dummy</b>"
895 895
896 896 def _repr_javascript_(self):
897 897 return "console.log('hi');", {'key': 'value'}
898 898
899 899
900 900 def test_user_variables():
901 901 # enable all formatters
902 902 ip.display_formatter.active_types = ip.display_formatter.format_types
903 903
904 904 ip.user_ns['dummy'] = d = DummyRepr()
905 905 keys = {'dummy', 'doesnotexist'}
906 906 r = ip.user_expressions({ key:key for key in keys})
907 907
908 908 assert keys == set(r.keys())
909 909 dummy = r["dummy"]
910 910 assert {"status", "data", "metadata"} == set(dummy.keys())
911 911 assert dummy["status"] == "ok"
912 912 data = dummy["data"]
913 913 metadata = dummy["metadata"]
914 914 assert data.get("text/html") == d._repr_html_()
915 915 js, jsmd = d._repr_javascript_()
916 916 assert data.get("application/javascript") == js
917 917 assert metadata.get("application/javascript") == jsmd
918 918
919 919 dne = r["doesnotexist"]
920 920 assert dne["status"] == "error"
921 921 assert dne["ename"] == "NameError"
922 922
923 923 # back to text only
924 924 ip.display_formatter.active_types = ['text/plain']
925 925
926 926 def test_user_expression():
927 927 # enable all formatters
928 928 ip.display_formatter.active_types = ip.display_formatter.format_types
929 929 query = {
930 930 'a' : '1 + 2',
931 931 'b' : '1/0',
932 932 }
933 933 r = ip.user_expressions(query)
934 934 import pprint
935 935 pprint.pprint(r)
936 936 assert set(r.keys()) == set(query.keys())
937 937 a = r["a"]
938 938 assert {"status", "data", "metadata"} == set(a.keys())
939 939 assert a["status"] == "ok"
940 940 data = a["data"]
941 941 metadata = a["metadata"]
942 942 assert data.get("text/plain") == "3"
943 943
944 944 b = r["b"]
945 945 assert b["status"] == "error"
946 946 assert b["ename"] == "ZeroDivisionError"
947 947
948 948 # back to text only
949 949 ip.display_formatter.active_types = ['text/plain']
950 950
951 951
952 952 class TestSyntaxErrorTransformer(unittest.TestCase):
953 953 """Check that SyntaxError raised by an input transformer is handled by run_cell()"""
954 954
955 955 @staticmethod
956 956 def transformer(lines):
957 957 for line in lines:
958 958 pos = line.find('syntaxerror')
959 959 if pos >= 0:
960 960 e = SyntaxError('input contains "syntaxerror"')
961 961 e.text = line
962 962 e.offset = pos + 1
963 963 raise e
964 964 return lines
965 965
966 966 def setUp(self):
967 967 ip.input_transformers_post.append(self.transformer)
968 968
969 969 def tearDown(self):
970 970 ip.input_transformers_post.remove(self.transformer)
971 971
972 972 def test_syntaxerror_input_transformer(self):
973 973 with tt.AssertPrints('1234'):
974 974 ip.run_cell('1234')
975 975 with tt.AssertPrints('SyntaxError: invalid syntax'):
976 976 ip.run_cell('1 2 3') # plain python syntax error
977 977 with tt.AssertPrints('SyntaxError: input contains "syntaxerror"'):
978 978 ip.run_cell('2345 # syntaxerror') # input transformer syntax error
979 979 with tt.AssertPrints('3456'):
980 980 ip.run_cell('3456')
981 981
982 982
983 983 class TestWarningSuppression(unittest.TestCase):
984 984 def test_warning_suppression(self):
985 985 ip.run_cell("import warnings")
986 986 try:
987 987 with self.assertWarnsRegex(UserWarning, "asdf"):
988 988 ip.run_cell("warnings.warn('asdf')")
989 989 # Here's the real test -- if we run that again, we should get the
990 990 # warning again. Traditionally, each warning was only issued once per
991 991 # IPython session (approximately), even if the user typed in new and
992 992 # different code that should have also triggered the warning, leading
993 993 # to much confusion.
994 994 with self.assertWarnsRegex(UserWarning, "asdf"):
995 995 ip.run_cell("warnings.warn('asdf')")
996 996 finally:
997 997 ip.run_cell("del warnings")
998 998
999 999
1000 1000 def test_deprecation_warning(self):
1001 1001 ip.run_cell("""
1002 1002 import warnings
1003 1003 def wrn():
1004 1004 warnings.warn(
1005 1005 "I AM A WARNING",
1006 1006 DeprecationWarning
1007 1007 )
1008 1008 """)
1009 1009 try:
1010 1010 with self.assertWarnsRegex(DeprecationWarning, "I AM A WARNING"):
1011 1011 ip.run_cell("wrn()")
1012 1012 finally:
1013 1013 ip.run_cell("del warnings")
1014 1014 ip.run_cell("del wrn")
1015 1015
1016 1016
1017 1017 class TestImportNoDeprecate(tt.TempFileMixin):
1018 1018
1019 1019 def setUp(self):
1020 1020 """Make a valid python temp file."""
1021 1021 self.mktmp("""
1022 1022 import warnings
1023 1023 def wrn():
1024 1024 warnings.warn(
1025 1025 "I AM A WARNING",
1026 1026 DeprecationWarning
1027 1027 )
1028 1028 """)
1029 1029 super().setUp()
1030 1030
1031 1031 def test_no_dep(self):
1032 1032 """
1033 1033 No deprecation warning should be raised from imported functions
1034 1034 """
1035 1035 ip.run_cell("from {} import wrn".format(self.fname))
1036 1036
1037 1037 with tt.AssertNotPrints("I AM A WARNING"):
1038 1038 ip.run_cell("wrn()")
1039 1039 ip.run_cell("del wrn")
1040 1040
1041 1041
1042 1042 def test_custom_exc_count():
1043 1043 hook = mock.Mock(return_value=None)
1044 1044 ip.set_custom_exc((SyntaxError,), hook)
1045 1045 before = ip.execution_count
1046 1046 ip.run_cell("def foo()", store_history=True)
1047 1047 # restore default excepthook
1048 1048 ip.set_custom_exc((), None)
1049 1049 assert hook.call_count == 1
1050 1050 assert ip.execution_count == before + 1
1051 1051
1052 1052
1053 1053 def test_run_cell_async():
1054 1054 ip.run_cell("import asyncio")
1055 1055 coro = ip.run_cell_async("await asyncio.sleep(0.01)\n5")
1056 1056 assert asyncio.iscoroutine(coro)
1057 1057 loop = asyncio.new_event_loop()
1058 1058 result = loop.run_until_complete(coro)
1059 1059 assert isinstance(result, interactiveshell.ExecutionResult)
1060 1060 assert result.result == 5
1061 1061
1062 1062
1063 1063 def test_run_cell_await():
1064 1064 ip.run_cell("import asyncio")
1065 1065 result = ip.run_cell("await asyncio.sleep(0.01); 10")
1066 1066 assert ip.user_ns["_"] == 10
1067 1067
1068 1068
1069 1069 def test_run_cell_asyncio_run():
1070 1070 ip.run_cell("import asyncio")
1071 1071 result = ip.run_cell("await asyncio.sleep(0.01); 1")
1072 1072 assert ip.user_ns["_"] == 1
1073 1073 result = ip.run_cell("asyncio.run(asyncio.sleep(0.01)); 2")
1074 1074 assert ip.user_ns["_"] == 2
1075 1075 result = ip.run_cell("await asyncio.sleep(0.01); 3")
1076 1076 assert ip.user_ns["_"] == 3
1077 1077
1078 1078
1079 1079 def test_should_run_async():
1080 1080 assert not ip.should_run_async("a = 5")
1081 1081 assert ip.should_run_async("await x")
1082 1082 assert ip.should_run_async("import asyncio; await asyncio.sleep(1)")
1083 1083
1084 1084
1085 1085 def test_set_custom_completer():
1086 1086 num_completers = len(ip.Completer.matchers)
1087 1087
1088 1088 def foo(*args, **kwargs):
1089 1089 return "I'm a completer!"
1090 1090
1091 1091 ip.set_custom_completer(foo, 0)
1092 1092
1093 1093 # check that we've really added a new completer
1094 1094 assert len(ip.Completer.matchers) == num_completers + 1
1095 1095
1096 1096 # check that the first completer is the function we defined
1097 1097 assert ip.Completer.matchers[0]() == "I'm a completer!"
1098 1098
1099 1099 # clean up
1100 1100 ip.Completer.custom_matchers.pop()
@@ -1,249 +1,250 b''
1 1 """Tests for the key interactiveshell module, where the main ipython class is defined.
2 2 """
3 3 #-----------------------------------------------------------------------------
4 4 # Module imports
5 5 #-----------------------------------------------------------------------------
6 6
7 7 # third party
8 8 import pytest
9 9
10 10 # our own packages
11 11
12 12 #-----------------------------------------------------------------------------
13 13 # Test functions
14 14 #-----------------------------------------------------------------------------
15 15
16 16 def test_reset():
17 17 """reset must clear most namespaces."""
18 18
19 19 # Check that reset runs without error
20 20 ip.reset()
21 21
22 22 # Once we've reset it (to clear of any junk that might have been there from
23 23 # other tests, we can count how many variables are in the user's namespace
24 24 nvars_user_ns = len(ip.user_ns)
25 25 nvars_hidden = len(ip.user_ns_hidden)
26 26
27 27 # Now add a few variables to user_ns, and check that reset clears them
28 28 ip.user_ns['x'] = 1
29 29 ip.user_ns['y'] = 1
30 30 ip.reset()
31 31
32 32 # Finally, check that all namespaces have only as many variables as we
33 33 # expect to find in them:
34 34 assert len(ip.user_ns) == nvars_user_ns
35 35 assert len(ip.user_ns_hidden) == nvars_hidden
36 36
37 37
38 38 # Tests for reporting of exceptions in various modes, handling of SystemExit,
39 39 # and %tb functionality. This is really a mix of testing ultraTB and interactiveshell.
40 40
41 41 def doctest_tb_plain():
42 42 """
43 43 In [18]: xmode plain
44 44 Exception reporting mode: Plain
45 45
46 46 In [19]: run simpleerr.py
47 47 Traceback (most recent call last):
48 48 File ...:... in <module>
49 49 bar(mode)
50 50 File ...:... in bar
51 51 div0()
52 52 File ...:... in div0
53 53 x/y
54 54 ZeroDivisionError: ...
55 55 """
56 56
57 57
58 58 def doctest_tb_context():
59 59 """
60 60 In [3]: xmode context
61 61 Exception reporting mode: Context
62 62
63 63 In [4]: run simpleerr.py
64 64 ---------------------------------------------------------------------------
65 65 ZeroDivisionError Traceback (most recent call last)
66 66 <BLANKLINE>
67 67 ... in <module>
68 68 30 except IndexError:
69 69 31 mode = 'div'
70 70 ---> 33 bar(mode)
71 71 <BLANKLINE>
72 72 ... in bar(mode)
73 73 15 "bar"
74 74 16 if mode=='div':
75 75 ---> 17 div0()
76 76 18 elif mode=='exit':
77 77 19 try:
78 78 <BLANKLINE>
79 79 ... in div0()
80 80 6 x = 1
81 81 7 y = 0
82 82 ----> 8 x/y
83 83 <BLANKLINE>
84 84 ZeroDivisionError: ..."""
85 85
86 86
87 87 def doctest_tb_verbose():
88 88 """
89 89 In [5]: xmode verbose
90 90 Exception reporting mode: Verbose
91 91
92 92 In [6]: run simpleerr.py
93 93 ---------------------------------------------------------------------------
94 94 ZeroDivisionError Traceback (most recent call last)
95 95 <BLANKLINE>
96 96 ... in <module>
97 97 30 except IndexError:
98 98 31 mode = 'div'
99 99 ---> 33 bar(mode)
100 100 mode = 'div'
101 101 <BLANKLINE>
102 102 ... in bar(mode='div')
103 103 15 "bar"
104 104 16 if mode=='div':
105 105 ---> 17 div0()
106 106 18 elif mode=='exit':
107 107 19 try:
108 108 <BLANKLINE>
109 109 ... in div0()
110 110 6 x = 1
111 111 7 y = 0
112 112 ----> 8 x/y
113 113 x = 1
114 114 y = 0
115 115 <BLANKLINE>
116 116 ZeroDivisionError: ...
117 117 """
118 118
119 119
120 120 def doctest_tb_sysexit():
121 121 """
122 122 In [17]: %xmode plain
123 123 Exception reporting mode: Plain
124 124
125 125 In [18]: %run simpleerr.py exit
126 126 An exception has occurred, use %tb to see the full traceback.
127 127 SystemExit: (1, 'Mode = exit')
128 128
129 129 In [19]: %run simpleerr.py exit 2
130 130 An exception has occurred, use %tb to see the full traceback.
131 131 SystemExit: (2, 'Mode = exit')
132 132
133 133 In [20]: %tb
134 134 Traceback (most recent call last):
135 135 File ...:... in execfile
136 136 exec(compiler(f.read(), fname, "exec"), glob, loc)
137 137 File ...:... in <module>
138 138 bar(mode)
139 139 File ...:... in bar
140 140 sysexit(stat, mode)
141 141 File ...:... in sysexit
142 142 raise SystemExit(stat, f"Mode = {mode}")
143 143 SystemExit: (2, 'Mode = exit')
144 144
145 145 In [21]: %xmode context
146 146 Exception reporting mode: Context
147 147
148 148 In [22]: %tb
149 149 ---------------------------------------------------------------------------
150 150 SystemExit Traceback (most recent call last)
151 151 File ..., in execfile(fname, glob, loc, compiler)
152 152 ... with open(fname, "rb") as f:
153 153 ... compiler = compiler or compile
154 154 ---> ... exec(compiler(f.read(), fname, "exec"), glob, loc)
155 155 ...<module>
156 156 30 except IndexError:
157 157 31 mode = 'div'
158 158 ---> 33 bar(mode)
159 159 <BLANKLINE>
160 160 ...bar(mode)
161 161 21 except:
162 162 22 stat = 1
163 163 ---> 23 sysexit(stat, mode)
164 164 24 else:
165 165 25 raise ValueError('Unknown mode')
166 166 <BLANKLINE>
167 167 ...sysexit(stat, mode)
168 168 10 def sysexit(stat, mode):
169 169 ---> 11 raise SystemExit(stat, f"Mode = {mode}")
170 170 <BLANKLINE>
171 171 SystemExit: (2, 'Mode = exit')
172 172 """
173 173
174 174
175 175 def doctest_tb_sysexit_verbose():
176 176 """
177 177 In [18]: %run simpleerr.py exit
178 178 An exception has occurred, use %tb to see the full traceback.
179 179 SystemExit: (1, 'Mode = exit')
180 180
181 181 In [19]: %run simpleerr.py exit 2
182 182 An exception has occurred, use %tb to see the full traceback.
183 183 SystemExit: (2, 'Mode = exit')
184 184
185 185 In [23]: %xmode verbose
186 186 Exception reporting mode: Verbose
187 187
188 188 In [24]: %tb
189 189 ---------------------------------------------------------------------------
190 190 SystemExit Traceback (most recent call last)
191 191 <BLANKLINE>
192 192 ... in <module>
193 193 30 except IndexError:
194 194 31 mode = 'div'
195 195 ---> 33 bar(mode)
196 196 mode = 'exit'
197 197 <BLANKLINE>
198 198 ... in bar(mode='exit')
199 199 ... except:
200 200 ... stat = 1
201 201 ---> ... sysexit(stat, mode)
202 202 mode = 'exit'
203 203 stat = 2
204 204 ... else:
205 205 ... raise ValueError('Unknown mode')
206 206 <BLANKLINE>
207 207 ... in sysexit(stat=2, mode='exit')
208 208 10 def sysexit(stat, mode):
209 209 ---> 11 raise SystemExit(stat, f"Mode = {mode}")
210 210 stat = 2
211 211 <BLANKLINE>
212 212 SystemExit: (2, 'Mode = exit')
213 213 """
214 214
215 215
216 216 def test_run_cell():
217 217 import textwrap
218 218
219 219 ip.run_cell("a = 10\na+=1")
220 220 ip.run_cell("assert a == 11\nassert 1")
221 221
222 222 assert ip.user_ns["a"] == 11
223 223 complex = textwrap.dedent(
224 224 """
225 225 if 1:
226 226 print "hello"
227 227 if 1:
228 228 print "world"
229 229
230 230 if 2:
231 231 print "foo"
232 232
233 233 if 3:
234 234 print "bar"
235 235
236 236 if 4:
237 237 print "bar"
238 238
239 """)
239 """
240 )
240 241 # Simply verifies that this kind of input is run
241 242 ip.run_cell(complex)
242 243
243 244
244 245 def test_db():
245 246 """Test the internal database used for variable persistence."""
246 247 ip.db["__unittest_"] = 12
247 248 assert ip.db["__unittest_"] == 12
248 249 del ip.db["__unittest_"]
249 250 assert "__unittest_" not in ip.db
@@ -1,1450 +1,1452 b''
1 1 # -*- coding: utf-8 -*-
2 2 """Tests for various magic functions."""
3 3
4 4 import asyncio
5 5 import gc
6 6 import io
7 7 import os
8 8 import re
9 9 import shlex
10 10 import sys
11 11 import warnings
12 12 from importlib import invalidate_caches
13 13 from io import StringIO
14 14 from pathlib import Path
15 15 from textwrap import dedent
16 16 from unittest import TestCase, mock
17 17
18 18 import pytest
19 19
20 20 from IPython import get_ipython
21 21 from IPython.core import magic
22 22 from IPython.core.error import UsageError
23 23 from IPython.core.magic import (
24 24 Magics,
25 25 cell_magic,
26 26 line_magic,
27 27 magics_class,
28 28 register_cell_magic,
29 29 register_line_magic,
30 30 )
31 31 from IPython.core.magics import code, execution, logging, osm, script
32 32 from IPython.testing import decorators as dec
33 33 from IPython.testing import tools as tt
34 34 from IPython.utils.io import capture_output
35 35 from IPython.utils.process import find_cmd
36 36 from IPython.utils.tempdir import TemporaryDirectory, TemporaryWorkingDirectory
37 37 from IPython.utils.syspathcontext import prepended_to_syspath
38 38
39 39 from .test_debugger import PdbTestInput
40 40
41 41 from tempfile import NamedTemporaryFile
42 42
43 43 @magic.magics_class
44 44 class DummyMagics(magic.Magics): pass
45 45
46 46 def test_extract_code_ranges():
47 47 instr = "1 3 5-6 7-9 10:15 17: :10 10- -13 :"
48 48 expected = [
49 49 (0, 1),
50 50 (2, 3),
51 51 (4, 6),
52 52 (6, 9),
53 53 (9, 14),
54 54 (16, None),
55 55 (None, 9),
56 56 (9, None),
57 57 (None, 13),
58 58 (None, None),
59 59 ]
60 60 actual = list(code.extract_code_ranges(instr))
61 61 assert actual == expected
62 62
63 63 def test_extract_symbols():
64 64 source = """import foo\na = 10\ndef b():\n return 42\n\n\nclass A: pass\n\n\n"""
65 65 symbols_args = ["a", "b", "A", "A,b", "A,a", "z"]
66 66 expected = [([], ['a']),
67 67 (["def b():\n return 42\n"], []),
68 68 (["class A: pass\n"], []),
69 69 (["class A: pass\n", "def b():\n return 42\n"], []),
70 70 (["class A: pass\n"], ['a']),
71 71 ([], ['z'])]
72 72 for symbols, exp in zip(symbols_args, expected):
73 73 assert code.extract_symbols(source, symbols) == exp
74 74
75 75
76 76 def test_extract_symbols_raises_exception_with_non_python_code():
77 77 source = ("=begin A Ruby program :)=end\n"
78 78 "def hello\n"
79 79 "puts 'Hello world'\n"
80 80 "end")
81 81 with pytest.raises(SyntaxError):
82 82 code.extract_symbols(source, "hello")
83 83
84 84
85 85 def test_magic_not_found():
86 86 # magic not found raises UsageError
87 87 with pytest.raises(UsageError):
88 88 _ip.magic('doesntexist')
89 89
90 90 # ensure result isn't success when a magic isn't found
91 91 result = _ip.run_cell('%doesntexist')
92 92 assert isinstance(result.error_in_exec, UsageError)
93 93
94 94
95 95 def test_cell_magic_not_found():
96 96 # magic not found raises UsageError
97 97 with pytest.raises(UsageError):
98 98 _ip.run_cell_magic('doesntexist', 'line', 'cell')
99 99
100 100 # ensure result isn't success when a magic isn't found
101 101 result = _ip.run_cell('%%doesntexist')
102 102 assert isinstance(result.error_in_exec, UsageError)
103 103
104 104
105 105 def test_magic_error_status():
106 106 def fail(shell):
107 107 1/0
108 108 _ip.register_magic_function(fail)
109 109 result = _ip.run_cell('%fail')
110 110 assert isinstance(result.error_in_exec, ZeroDivisionError)
111 111
112 112
113 113 def test_config():
114 114 """ test that config magic does not raise
115 115 can happen if Configurable init is moved too early into
116 116 Magics.__init__ as then a Config object will be registered as a
117 117 magic.
118 118 """
119 119 ## should not raise.
120 120 _ip.magic('config')
121 121
122 122 def test_config_available_configs():
123 123 """ test that config magic prints available configs in unique and
124 124 sorted order. """
125 125 with capture_output() as captured:
126 126 _ip.magic('config')
127 127
128 128 stdout = captured.stdout
129 129 config_classes = stdout.strip().split('\n')[1:]
130 130 assert config_classes == sorted(set(config_classes))
131 131
132 132 def test_config_print_class():
133 133 """ test that config with a classname prints the class's options. """
134 134 with capture_output() as captured:
135 135 _ip.magic('config TerminalInteractiveShell')
136 136
137 137 stdout = captured.stdout
138 138 assert re.match(
139 139 "TerminalInteractiveShell.* options", stdout.splitlines()[0]
140 140 ), f"{stdout}\n\n1st line of stdout not like 'TerminalInteractiveShell.* options'"
141 141
142 142
143 143 def test_rehashx():
144 144 # clear up everything
145 145 _ip.alias_manager.clear_aliases()
146 146 del _ip.db['syscmdlist']
147 147
148 148 _ip.magic('rehashx')
149 149 # Practically ALL ipython development systems will have more than 10 aliases
150 150
151 151 assert len(_ip.alias_manager.aliases) > 10
152 152 for name, cmd in _ip.alias_manager.aliases:
153 153 # we must strip dots from alias names
154 154 assert "." not in name
155 155
156 156 # rehashx must fill up syscmdlist
157 157 scoms = _ip.db['syscmdlist']
158 158 assert len(scoms) > 10
159 159
160 160
161 161 def test_magic_parse_options():
162 162 """Test that we don't mangle paths when parsing magic options."""
163 163 ip = get_ipython()
164 164 path = 'c:\\x'
165 165 m = DummyMagics(ip)
166 166 opts = m.parse_options('-f %s' % path,'f:')[0]
167 167 # argv splitting is os-dependent
168 168 if os.name == 'posix':
169 169 expected = 'c:x'
170 170 else:
171 171 expected = path
172 172 assert opts["f"] == expected
173 173
174 174
175 175 def test_magic_parse_long_options():
176 176 """Magic.parse_options can handle --foo=bar long options"""
177 177 ip = get_ipython()
178 178 m = DummyMagics(ip)
179 179 opts, _ = m.parse_options("--foo --bar=bubble", "a", "foo", "bar=")
180 180 assert "foo" in opts
181 181 assert "bar" in opts
182 182 assert opts["bar"] == "bubble"
183 183
184 184
185 185 def doctest_hist_f():
186 186 """Test %hist -f with temporary filename.
187 187
188 188 In [9]: import tempfile
189 189
190 190 In [10]: tfile = tempfile.mktemp('.py','tmp-ipython-')
191 191
192 192 In [11]: %hist -nl -f $tfile 3
193 193
194 194 In [13]: import os; os.unlink(tfile)
195 195 """
196 196
197 197
198 198 def doctest_hist_op():
199 199 """Test %hist -op
200 200
201 201 In [1]: class b(float):
202 202 ...: pass
203 203 ...:
204 204
205 205 In [2]: class s(object):
206 206 ...: def __str__(self):
207 207 ...: return 's'
208 208 ...:
209 209
210 210 In [3]:
211 211
212 212 In [4]: class r(b):
213 213 ...: def __repr__(self):
214 214 ...: return 'r'
215 215 ...:
216 216
217 217 In [5]: class sr(s,r): pass
218 218 ...:
219 219
220 220 In [6]:
221 221
222 222 In [7]: bb=b()
223 223
224 224 In [8]: ss=s()
225 225
226 226 In [9]: rr=r()
227 227
228 228 In [10]: ssrr=sr()
229 229
230 230 In [11]: 4.5
231 231 Out[11]: 4.5
232 232
233 233 In [12]: str(ss)
234 234 Out[12]: 's'
235 235
236 236 In [13]:
237 237
238 238 In [14]: %hist -op
239 239 >>> class b:
240 240 ... pass
241 241 ...
242 242 >>> class s(b):
243 243 ... def __str__(self):
244 244 ... return 's'
245 245 ...
246 246 >>>
247 247 >>> class r(b):
248 248 ... def __repr__(self):
249 249 ... return 'r'
250 250 ...
251 251 >>> class sr(s,r): pass
252 252 >>>
253 253 >>> bb=b()
254 254 >>> ss=s()
255 255 >>> rr=r()
256 256 >>> ssrr=sr()
257 257 >>> 4.5
258 258 4.5
259 259 >>> str(ss)
260 260 's'
261 261 >>>
262 262 """
263 263
264 264 def test_hist_pof():
265 265 ip = get_ipython()
266 266 ip.run_cell("1+2", store_history=True)
267 267 #raise Exception(ip.history_manager.session_number)
268 268 #raise Exception(list(ip.history_manager._get_range_session()))
269 269 with TemporaryDirectory() as td:
270 270 tf = os.path.join(td, 'hist.py')
271 271 ip.run_line_magic('history', '-pof %s' % tf)
272 272 assert os.path.isfile(tf)
273 273
274 274
275 275 def test_macro():
276 276 ip = get_ipython()
277 277 ip.history_manager.reset() # Clear any existing history.
278 278 cmds = ["a=1", "def b():\n return a**2", "print(a,b())"]
279 279 for i, cmd in enumerate(cmds, start=1):
280 280 ip.history_manager.store_inputs(i, cmd)
281 281 ip.magic("macro test 1-3")
282 282 assert ip.user_ns["test"].value == "\n".join(cmds) + "\n"
283 283
284 284 # List macros
285 285 assert "test" in ip.magic("macro")
286 286
287 287
288 288 def test_macro_run():
289 289 """Test that we can run a multi-line macro successfully."""
290 290 ip = get_ipython()
291 291 ip.history_manager.reset()
292 292 cmds = ["a=10", "a+=1", "print(a)", "%macro test 2-3"]
293 293 for cmd in cmds:
294 294 ip.run_cell(cmd, store_history=True)
295 295 assert ip.user_ns["test"].value == "a+=1\nprint(a)\n"
296 296 with tt.AssertPrints("12"):
297 297 ip.run_cell("test")
298 298 with tt.AssertPrints("13"):
299 299 ip.run_cell("test")
300 300
301 301
302 302 def test_magic_magic():
303 303 """Test %magic"""
304 304 ip = get_ipython()
305 305 with capture_output() as captured:
306 306 ip.magic("magic")
307 307
308 308 stdout = captured.stdout
309 309 assert "%magic" in stdout
310 310 assert "IPython" in stdout
311 311 assert "Available" in stdout
312 312
313 313
314 314 @dec.skipif_not_numpy
315 315 def test_numpy_reset_array_undec():
316 316 "Test '%reset array' functionality"
317 317 _ip.ex("import numpy as np")
318 318 _ip.ex("a = np.empty(2)")
319 319 assert "a" in _ip.user_ns
320 320 _ip.magic("reset -f array")
321 321 assert "a" not in _ip.user_ns
322 322
323 323
324 324 def test_reset_out():
325 325 "Test '%reset out' magic"
326 326 _ip.run_cell("parrot = 'dead'", store_history=True)
327 327 # test '%reset -f out', make an Out prompt
328 328 _ip.run_cell("parrot", store_history=True)
329 329 assert "dead" in [_ip.user_ns[x] for x in ("_", "__", "___")]
330 330 _ip.magic("reset -f out")
331 331 assert "dead" not in [_ip.user_ns[x] for x in ("_", "__", "___")]
332 332 assert len(_ip.user_ns["Out"]) == 0
333 333
334 334
335 335 def test_reset_in():
336 336 "Test '%reset in' magic"
337 337 # test '%reset -f in'
338 338 _ip.run_cell("parrot", store_history=True)
339 339 assert "parrot" in [_ip.user_ns[x] for x in ("_i", "_ii", "_iii")]
340 340 _ip.magic("%reset -f in")
341 341 assert "parrot" not in [_ip.user_ns[x] for x in ("_i", "_ii", "_iii")]
342 342 assert len(set(_ip.user_ns["In"])) == 1
343 343
344 344
345 345 def test_reset_dhist():
346 346 "Test '%reset dhist' magic"
347 347 _ip.run_cell("tmp = [d for d in _dh]") # copy before clearing
348 348 _ip.magic("cd " + os.path.dirname(pytest.__file__))
349 349 _ip.magic("cd -")
350 350 assert len(_ip.user_ns["_dh"]) > 0
351 351 _ip.magic("reset -f dhist")
352 352 assert len(_ip.user_ns["_dh"]) == 0
353 353 _ip.run_cell("_dh = [d for d in tmp]") # restore
354 354
355 355
356 356 def test_reset_in_length():
357 357 "Test that '%reset in' preserves In[] length"
358 358 _ip.run_cell("print 'foo'")
359 359 _ip.run_cell("reset -f in")
360 360 assert len(_ip.user_ns["In"]) == _ip.displayhook.prompt_count + 1
361 361
362 362
363 363 class TestResetErrors(TestCase):
364 364
365 365 def test_reset_redefine(self):
366 366
367 367 @magics_class
368 368 class KernelMagics(Magics):
369 369 @line_magic
370 370 def less(self, shell): pass
371 371
372 372 _ip.register_magics(KernelMagics)
373 373
374 374 with self.assertLogs() as cm:
375 375 # hack, we want to just capture logs, but assertLogs fails if not
376 376 # logs get produce.
377 377 # so log one things we ignore.
378 378 import logging as log_mod
379 379 log = log_mod.getLogger()
380 380 log.info('Nothing')
381 381 # end hack.
382 382 _ip.run_cell("reset -f")
383 383
384 384 assert len(cm.output) == 1
385 385 for out in cm.output:
386 386 assert "Invalid alias" not in out
387 387
388 388 def test_tb_syntaxerror():
389 389 """test %tb after a SyntaxError"""
390 390 ip = get_ipython()
391 391 ip.run_cell("for")
392 392
393 393 # trap and validate stdout
394 394 save_stdout = sys.stdout
395 395 try:
396 396 sys.stdout = StringIO()
397 397 ip.run_cell("%tb")
398 398 out = sys.stdout.getvalue()
399 399 finally:
400 400 sys.stdout = save_stdout
401 401 # trim output, and only check the last line
402 402 last_line = out.rstrip().splitlines()[-1].strip()
403 403 assert last_line == "SyntaxError: invalid syntax"
404 404
405 405
406 406 def test_time():
407 407 ip = get_ipython()
408 408
409 409 with tt.AssertPrints("Wall time: "):
410 410 ip.run_cell("%time None")
411 411
412 412 ip.run_cell("def f(kmjy):\n"
413 413 " %time print (2*kmjy)")
414 414
415 415 with tt.AssertPrints("Wall time: "):
416 416 with tt.AssertPrints("hihi", suppress=False):
417 417 ip.run_cell("f('hi')")
418 418
419 419 def test_time_last_not_expression():
420 420 ip.run_cell("%%time\n"
421 421 "var_1 = 1\n"
422 422 "var_2 = 2\n")
423 423 assert ip.user_ns['var_1'] == 1
424 424 del ip.user_ns['var_1']
425 425 assert ip.user_ns['var_2'] == 2
426 426 del ip.user_ns['var_2']
427 427
428 428
429 429 @dec.skip_win32
430 430 def test_time2():
431 431 ip = get_ipython()
432 432
433 433 with tt.AssertPrints("CPU times: user "):
434 434 ip.run_cell("%time None")
435 435
436 436 def test_time3():
437 437 """Erroneous magic function calls, issue gh-3334"""
438 438 ip = get_ipython()
439 439 ip.user_ns.pop('run', None)
440 440
441 441 with tt.AssertNotPrints("not found", channel='stderr'):
442 442 ip.run_cell("%%time\n"
443 443 "run = 0\n"
444 444 "run += 1")
445 445
446 446 def test_multiline_time():
447 447 """Make sure last statement from time return a value."""
448 448 ip = get_ipython()
449 449 ip.user_ns.pop('run', None)
450 450
451 ip.run_cell(dedent("""\
451 ip.run_cell(
452 dedent(
453 """\
452 454 %%time
453 455 a = "ho"
454 456 b = "hey"
455 457 a+b
456 458 """
457 459 )
458 460 )
459 461 assert ip.user_ns_hidden["_"] == "hohey"
460 462
461 463
462 464 def test_time_local_ns():
463 465 """
464 466 Test that local_ns is actually global_ns when running a cell magic
465 467 """
466 468 ip = get_ipython()
467 469 ip.run_cell("%%time\n" "myvar = 1")
468 470 assert ip.user_ns["myvar"] == 1
469 471 del ip.user_ns["myvar"]
470 472
471 473
472 474 def test_doctest_mode():
473 475 "Toggle doctest_mode twice, it should be a no-op and run without error"
474 476 _ip.magic('doctest_mode')
475 477 _ip.magic('doctest_mode')
476 478
477 479
478 480 def test_parse_options():
479 481 """Tests for basic options parsing in magics."""
480 482 # These are only the most minimal of tests, more should be added later. At
481 483 # the very least we check that basic text/unicode calls work OK.
482 484 m = DummyMagics(_ip)
483 485 assert m.parse_options("foo", "")[1] == "foo"
484 486 assert m.parse_options("foo", "")[1] == "foo"
485 487
486 488
487 489 def test_parse_options_preserve_non_option_string():
488 490 """Test to assert preservation of non-option part of magic-block, while parsing magic options."""
489 491 m = DummyMagics(_ip)
490 492 opts, stmt = m.parse_options(
491 493 " -n1 -r 13 _ = 314 + foo", "n:r:", preserve_non_opts=True
492 494 )
493 495 assert opts == {"n": "1", "r": "13"}
494 496 assert stmt == "_ = 314 + foo"
495 497
496 498
497 499 def test_run_magic_preserve_code_block():
498 500 """Test to assert preservation of non-option part of magic-block, while running magic."""
499 501 _ip.user_ns["spaces"] = []
500 502 _ip.magic("timeit -n1 -r1 spaces.append([s.count(' ') for s in ['document']])")
501 503 assert _ip.user_ns["spaces"] == [[0]]
502 504
503 505
504 506 def test_dirops():
505 507 """Test various directory handling operations."""
506 508 # curpath = lambda :os.path.splitdrive(os.getcwd())[1].replace('\\','/')
507 509 curpath = os.getcwd
508 510 startdir = os.getcwd()
509 511 ipdir = os.path.realpath(_ip.ipython_dir)
510 512 try:
511 513 _ip.magic('cd "%s"' % ipdir)
512 514 assert curpath() == ipdir
513 515 _ip.magic('cd -')
514 516 assert curpath() == startdir
515 517 _ip.magic('pushd "%s"' % ipdir)
516 518 assert curpath() == ipdir
517 519 _ip.magic('popd')
518 520 assert curpath() == startdir
519 521 finally:
520 522 os.chdir(startdir)
521 523
522 524
523 525 def test_cd_force_quiet():
524 526 """Test OSMagics.cd_force_quiet option"""
525 527 _ip.config.OSMagics.cd_force_quiet = True
526 528 osmagics = osm.OSMagics(shell=_ip)
527 529
528 530 startdir = os.getcwd()
529 531 ipdir = os.path.realpath(_ip.ipython_dir)
530 532
531 533 try:
532 534 with tt.AssertNotPrints(ipdir):
533 535 osmagics.cd('"%s"' % ipdir)
534 536 with tt.AssertNotPrints(startdir):
535 537 osmagics.cd('-')
536 538 finally:
537 539 os.chdir(startdir)
538 540
539 541
540 542 def test_xmode():
541 543 # Calling xmode three times should be a no-op
542 544 xmode = _ip.InteractiveTB.mode
543 545 for i in range(4):
544 546 _ip.magic("xmode")
545 547 assert _ip.InteractiveTB.mode == xmode
546 548
547 549 def test_reset_hard():
548 550 monitor = []
549 551 class A(object):
550 552 def __del__(self):
551 553 monitor.append(1)
552 554 def __repr__(self):
553 555 return "<A instance>"
554 556
555 557 _ip.user_ns["a"] = A()
556 558 _ip.run_cell("a")
557 559
558 560 assert monitor == []
559 561 _ip.magic("reset -f")
560 562 assert monitor == [1]
561 563
562 564 class TestXdel(tt.TempFileMixin):
563 565 def test_xdel(self):
564 566 """Test that references from %run are cleared by xdel."""
565 567 src = ("class A(object):\n"
566 568 " monitor = []\n"
567 569 " def __del__(self):\n"
568 570 " self.monitor.append(1)\n"
569 571 "a = A()\n")
570 572 self.mktmp(src)
571 573 # %run creates some hidden references...
572 574 _ip.magic("run %s" % self.fname)
573 575 # ... as does the displayhook.
574 576 _ip.run_cell("a")
575 577
576 578 monitor = _ip.user_ns["A"].monitor
577 579 assert monitor == []
578 580
579 581 _ip.magic("xdel a")
580 582
581 583 # Check that a's __del__ method has been called.
582 584 gc.collect(0)
583 585 assert monitor == [1]
584 586
585 587 def doctest_who():
586 588 """doctest for %who
587 589
588 590 In [1]: %reset -sf
589 591
590 592 In [2]: alpha = 123
591 593
592 594 In [3]: beta = 'beta'
593 595
594 596 In [4]: %who int
595 597 alpha
596 598
597 599 In [5]: %who str
598 600 beta
599 601
600 602 In [6]: %whos
601 603 Variable Type Data/Info
602 604 ----------------------------
603 605 alpha int 123
604 606 beta str beta
605 607
606 608 In [7]: %who_ls
607 609 Out[7]: ['alpha', 'beta']
608 610 """
609 611
610 612 def test_whos():
611 613 """Check that whos is protected against objects where repr() fails."""
612 614 class A(object):
613 615 def __repr__(self):
614 616 raise Exception()
615 617 _ip.user_ns['a'] = A()
616 618 _ip.magic("whos")
617 619
618 620 def doctest_precision():
619 621 """doctest for %precision
620 622
621 623 In [1]: f = get_ipython().display_formatter.formatters['text/plain']
622 624
623 625 In [2]: %precision 5
624 626 Out[2]: '%.5f'
625 627
626 628 In [3]: f.float_format
627 629 Out[3]: '%.5f'
628 630
629 631 In [4]: %precision %e
630 632 Out[4]: '%e'
631 633
632 634 In [5]: f(3.1415927)
633 635 Out[5]: '3.141593e+00'
634 636 """
635 637
636 638 def test_debug_magic():
637 639 """Test debugging a small code with %debug
638 640
639 641 In [1]: with PdbTestInput(['c']):
640 642 ...: %debug print("a b") #doctest: +ELLIPSIS
641 643 ...:
642 644 ...
643 645 ipdb> c
644 646 a b
645 647 In [2]:
646 648 """
647 649
648 650 def test_psearch():
649 651 with tt.AssertPrints("dict.fromkeys"):
650 652 _ip.run_cell("dict.fr*?")
651 653 with tt.AssertPrints("π.is_integer"):
652 654 _ip.run_cell("π = 3.14;\nπ.is_integ*?")
653 655
654 656 def test_timeit_shlex():
655 657 """test shlex issues with timeit (#1109)"""
656 658 _ip.ex("def f(*a,**kw): pass")
657 659 _ip.magic('timeit -n1 "this is a bug".count(" ")')
658 660 _ip.magic('timeit -r1 -n1 f(" ", 1)')
659 661 _ip.magic('timeit -r1 -n1 f(" ", 1, " ", 2, " ")')
660 662 _ip.magic('timeit -r1 -n1 ("a " + "b")')
661 663 _ip.magic('timeit -r1 -n1 f("a " + "b")')
662 664 _ip.magic('timeit -r1 -n1 f("a " + "b ")')
663 665
664 666
665 667 def test_timeit_special_syntax():
666 668 "Test %%timeit with IPython special syntax"
667 669 @register_line_magic
668 670 def lmagic(line):
669 671 ip = get_ipython()
670 672 ip.user_ns['lmagic_out'] = line
671 673
672 674 # line mode test
673 675 _ip.run_line_magic("timeit", "-n1 -r1 %lmagic my line")
674 676 assert _ip.user_ns["lmagic_out"] == "my line"
675 677 # cell mode test
676 678 _ip.run_cell_magic("timeit", "-n1 -r1", "%lmagic my line2")
677 679 assert _ip.user_ns["lmagic_out"] == "my line2"
678 680
679 681
680 682 def test_timeit_return():
681 683 """
682 684 test whether timeit -o return object
683 685 """
684 686
685 687 res = _ip.run_line_magic('timeit','-n10 -r10 -o 1')
686 688 assert(res is not None)
687 689
688 690 def test_timeit_quiet():
689 691 """
690 692 test quiet option of timeit magic
691 693 """
692 694 with tt.AssertNotPrints("loops"):
693 695 _ip.run_cell("%timeit -n1 -r1 -q 1")
694 696
695 697 def test_timeit_return_quiet():
696 698 with tt.AssertNotPrints("loops"):
697 699 res = _ip.run_line_magic('timeit', '-n1 -r1 -q -o 1')
698 700 assert (res is not None)
699 701
700 702 def test_timeit_invalid_return():
701 703 with pytest.raises(SyntaxError):
702 704 _ip.run_line_magic('timeit', 'return')
703 705
704 706 @dec.skipif(execution.profile is None)
705 707 def test_prun_special_syntax():
706 708 "Test %%prun with IPython special syntax"
707 709 @register_line_magic
708 710 def lmagic(line):
709 711 ip = get_ipython()
710 712 ip.user_ns['lmagic_out'] = line
711 713
712 714 # line mode test
713 715 _ip.run_line_magic("prun", "-q %lmagic my line")
714 716 assert _ip.user_ns["lmagic_out"] == "my line"
715 717 # cell mode test
716 718 _ip.run_cell_magic("prun", "-q", "%lmagic my line2")
717 719 assert _ip.user_ns["lmagic_out"] == "my line2"
718 720
719 721
720 722 @dec.skipif(execution.profile is None)
721 723 def test_prun_quotes():
722 724 "Test that prun does not clobber string escapes (GH #1302)"
723 725 _ip.magic(r"prun -q x = '\t'")
724 726 assert _ip.user_ns["x"] == "\t"
725 727
726 728
727 729 def test_extension():
728 730 # Debugging information for failures of this test
729 731 print('sys.path:')
730 732 for p in sys.path:
731 733 print(' ', p)
732 734 print('CWD', os.getcwd())
733 735
734 736 pytest.raises(ImportError, _ip.magic, "load_ext daft_extension")
735 737 daft_path = os.path.join(os.path.dirname(__file__), "daft_extension")
736 738 sys.path.insert(0, daft_path)
737 739 try:
738 740 _ip.user_ns.pop('arq', None)
739 741 invalidate_caches() # Clear import caches
740 742 _ip.magic("load_ext daft_extension")
741 743 assert _ip.user_ns["arq"] == 185
742 744 _ip.magic("unload_ext daft_extension")
743 745 assert 'arq' not in _ip.user_ns
744 746 finally:
745 747 sys.path.remove(daft_path)
746 748
747 749
748 750 def test_notebook_export_json():
749 751 pytest.importorskip("nbformat")
750 752 _ip = get_ipython()
751 753 _ip.history_manager.reset() # Clear any existing history.
752 754 cmds = ["a=1", "def b():\n return a**2", "print('noël, été', b())"]
753 755 for i, cmd in enumerate(cmds, start=1):
754 756 _ip.history_manager.store_inputs(i, cmd)
755 757 with TemporaryDirectory() as td:
756 758 outfile = os.path.join(td, "nb.ipynb")
757 759 _ip.magic("notebook %s" % outfile)
758 760
759 761
760 762 class TestEnv(TestCase):
761 763
762 764 def test_env(self):
763 765 env = _ip.magic("env")
764 766 self.assertTrue(isinstance(env, dict))
765 767
766 768 def test_env_secret(self):
767 769 env = _ip.magic("env")
768 770 hidden = "<hidden>"
769 771 with mock.patch.dict(
770 772 os.environ,
771 773 {
772 774 "API_KEY": "abc123",
773 775 "SECRET_THING": "ssshhh",
774 776 "JUPYTER_TOKEN": "",
775 777 "VAR": "abc"
776 778 }
777 779 ):
778 780 env = _ip.magic("env")
779 781 assert env["API_KEY"] == hidden
780 782 assert env["SECRET_THING"] == hidden
781 783 assert env["JUPYTER_TOKEN"] == hidden
782 784 assert env["VAR"] == "abc"
783 785
784 786 def test_env_get_set_simple(self):
785 787 env = _ip.magic("env var val1")
786 788 self.assertEqual(env, None)
787 789 self.assertEqual(os.environ['var'], 'val1')
788 790 self.assertEqual(_ip.magic("env var"), 'val1')
789 791 env = _ip.magic("env var=val2")
790 792 self.assertEqual(env, None)
791 793 self.assertEqual(os.environ['var'], 'val2')
792 794
793 795 def test_env_get_set_complex(self):
794 796 env = _ip.magic("env var 'val1 '' 'val2")
795 797 self.assertEqual(env, None)
796 798 self.assertEqual(os.environ['var'], "'val1 '' 'val2")
797 799 self.assertEqual(_ip.magic("env var"), "'val1 '' 'val2")
798 800 env = _ip.magic('env var=val2 val3="val4')
799 801 self.assertEqual(env, None)
800 802 self.assertEqual(os.environ['var'], 'val2 val3="val4')
801 803
802 804 def test_env_set_bad_input(self):
803 805 self.assertRaises(UsageError, lambda: _ip.magic("set_env var"))
804 806
805 807 def test_env_set_whitespace(self):
806 808 self.assertRaises(UsageError, lambda: _ip.magic("env var A=B"))
807 809
808 810
809 811 class CellMagicTestCase(TestCase):
810 812
811 813 def check_ident(self, magic):
812 814 # Manually called, we get the result
813 815 out = _ip.run_cell_magic(magic, "a", "b")
814 816 assert out == ("a", "b")
815 817 # Via run_cell, it goes into the user's namespace via displayhook
816 818 _ip.run_cell("%%" + magic + " c\nd\n")
817 819 assert _ip.user_ns["_"] == ("c", "d\n")
818 820
819 821 def test_cell_magic_func_deco(self):
820 822 "Cell magic using simple decorator"
821 823 @register_cell_magic
822 824 def cellm(line, cell):
823 825 return line, cell
824 826
825 827 self.check_ident('cellm')
826 828
827 829 def test_cell_magic_reg(self):
828 830 "Cell magic manually registered"
829 831 def cellm(line, cell):
830 832 return line, cell
831 833
832 834 _ip.register_magic_function(cellm, 'cell', 'cellm2')
833 835 self.check_ident('cellm2')
834 836
835 837 def test_cell_magic_class(self):
836 838 "Cell magics declared via a class"
837 839 @magics_class
838 840 class MyMagics(Magics):
839 841
840 842 @cell_magic
841 843 def cellm3(self, line, cell):
842 844 return line, cell
843 845
844 846 _ip.register_magics(MyMagics)
845 847 self.check_ident('cellm3')
846 848
847 849 def test_cell_magic_class2(self):
848 850 "Cell magics declared via a class, #2"
849 851 @magics_class
850 852 class MyMagics2(Magics):
851 853
852 854 @cell_magic('cellm4')
853 855 def cellm33(self, line, cell):
854 856 return line, cell
855 857
856 858 _ip.register_magics(MyMagics2)
857 859 self.check_ident('cellm4')
858 860 # Check that nothing is registered as 'cellm33'
859 861 c33 = _ip.find_cell_magic('cellm33')
860 862 assert c33 == None
861 863
862 864 def test_file():
863 865 """Basic %%writefile"""
864 866 ip = get_ipython()
865 867 with TemporaryDirectory() as td:
866 868 fname = os.path.join(td, "file1")
867 869 ip.run_cell_magic(
868 870 "writefile",
869 871 fname,
870 872 "\n".join(
871 873 [
872 874 "line1",
873 875 "line2",
874 876 ]
875 877 ),
876 878 )
877 879 s = Path(fname).read_text(encoding="utf-8")
878 880 assert "line1\n" in s
879 881 assert "line2" in s
880 882
881 883
882 884 @dec.skip_win32
883 885 def test_file_single_quote():
884 886 """Basic %%writefile with embedded single quotes"""
885 887 ip = get_ipython()
886 888 with TemporaryDirectory() as td:
887 889 fname = os.path.join(td, "'file1'")
888 890 ip.run_cell_magic(
889 891 "writefile",
890 892 fname,
891 893 "\n".join(
892 894 [
893 895 "line1",
894 896 "line2",
895 897 ]
896 898 ),
897 899 )
898 900 s = Path(fname).read_text(encoding="utf-8")
899 901 assert "line1\n" in s
900 902 assert "line2" in s
901 903
902 904
903 905 @dec.skip_win32
904 906 def test_file_double_quote():
905 907 """Basic %%writefile with embedded double quotes"""
906 908 ip = get_ipython()
907 909 with TemporaryDirectory() as td:
908 910 fname = os.path.join(td, '"file1"')
909 911 ip.run_cell_magic(
910 912 "writefile",
911 913 fname,
912 914 "\n".join(
913 915 [
914 916 "line1",
915 917 "line2",
916 918 ]
917 919 ),
918 920 )
919 921 s = Path(fname).read_text(encoding="utf-8")
920 922 assert "line1\n" in s
921 923 assert "line2" in s
922 924
923 925
924 926 def test_file_var_expand():
925 927 """%%writefile $filename"""
926 928 ip = get_ipython()
927 929 with TemporaryDirectory() as td:
928 930 fname = os.path.join(td, "file1")
929 931 ip.user_ns["filename"] = fname
930 932 ip.run_cell_magic(
931 933 "writefile",
932 934 "$filename",
933 935 "\n".join(
934 936 [
935 937 "line1",
936 938 "line2",
937 939 ]
938 940 ),
939 941 )
940 942 s = Path(fname).read_text(encoding="utf-8")
941 943 assert "line1\n" in s
942 944 assert "line2" in s
943 945
944 946
945 947 def test_file_unicode():
946 948 """%%writefile with unicode cell"""
947 949 ip = get_ipython()
948 950 with TemporaryDirectory() as td:
949 951 fname = os.path.join(td, 'file1')
950 952 ip.run_cell_magic("writefile", fname, u'\n'.join([
951 953 u'liné1',
952 954 u'liné2',
953 955 ]))
954 956 with io.open(fname, encoding='utf-8') as f:
955 957 s = f.read()
956 958 assert "liné1\n" in s
957 959 assert "liné2" in s
958 960
959 961
960 962 def test_file_amend():
961 963 """%%writefile -a amends files"""
962 964 ip = get_ipython()
963 965 with TemporaryDirectory() as td:
964 966 fname = os.path.join(td, "file2")
965 967 ip.run_cell_magic(
966 968 "writefile",
967 969 fname,
968 970 "\n".join(
969 971 [
970 972 "line1",
971 973 "line2",
972 974 ]
973 975 ),
974 976 )
975 977 ip.run_cell_magic(
976 978 "writefile",
977 979 "-a %s" % fname,
978 980 "\n".join(
979 981 [
980 982 "line3",
981 983 "line4",
982 984 ]
983 985 ),
984 986 )
985 987 s = Path(fname).read_text(encoding="utf-8")
986 988 assert "line1\n" in s
987 989 assert "line3\n" in s
988 990
989 991
990 992 def test_file_spaces():
991 993 """%%file with spaces in filename"""
992 994 ip = get_ipython()
993 995 with TemporaryWorkingDirectory() as td:
994 996 fname = "file name"
995 997 ip.run_cell_magic(
996 998 "file",
997 999 '"%s"' % fname,
998 1000 "\n".join(
999 1001 [
1000 1002 "line1",
1001 1003 "line2",
1002 1004 ]
1003 1005 ),
1004 1006 )
1005 1007 s = Path(fname).read_text(encoding="utf-8")
1006 1008 assert "line1\n" in s
1007 1009 assert "line2" in s
1008 1010
1009 1011
1010 1012 def test_script_config():
1011 1013 ip = get_ipython()
1012 1014 ip.config.ScriptMagics.script_magics = ['whoda']
1013 1015 sm = script.ScriptMagics(shell=ip)
1014 1016 assert "whoda" in sm.magics["cell"]
1015 1017
1016 1018
1017 1019 def test_script_out():
1018 1020 ip = get_ipython()
1019 1021 ip.run_cell_magic("script", f"--out output {sys.executable}", "print('hi')")
1020 1022 assert ip.user_ns["output"].strip() == "hi"
1021 1023
1022 1024
1023 1025 def test_script_err():
1024 1026 ip = get_ipython()
1025 1027 ip.run_cell_magic(
1026 1028 "script",
1027 1029 f"--err error {sys.executable}",
1028 1030 "import sys; print('hello', file=sys.stderr)",
1029 1031 )
1030 1032 assert ip.user_ns["error"].strip() == "hello"
1031 1033
1032 1034
1033 1035 def test_script_out_err():
1034 1036
1035 1037 ip = get_ipython()
1036 1038 ip.run_cell_magic(
1037 1039 "script",
1038 1040 f"--out output --err error {sys.executable}",
1039 1041 "\n".join(
1040 1042 [
1041 1043 "import sys",
1042 1044 "print('hi')",
1043 1045 "print('hello', file=sys.stderr)",
1044 1046 ]
1045 1047 ),
1046 1048 )
1047 1049 assert ip.user_ns["output"].strip() == "hi"
1048 1050 assert ip.user_ns["error"].strip() == "hello"
1049 1051
1050 1052
1051 1053 async def test_script_bg_out():
1052 1054 ip = get_ipython()
1053 1055 ip.run_cell_magic("script", f"--bg --out output {sys.executable}", "print('hi')")
1054 1056 assert (await ip.user_ns["output"].read()).strip() == b"hi"
1055 1057 assert ip.user_ns["output"].at_eof()
1056 1058
1057 1059
1058 1060 async def test_script_bg_err():
1059 1061 ip = get_ipython()
1060 1062 ip.run_cell_magic(
1061 1063 "script",
1062 1064 f"--bg --err error {sys.executable}",
1063 1065 "import sys; print('hello', file=sys.stderr)",
1064 1066 )
1065 1067 assert (await ip.user_ns["error"].read()).strip() == b"hello"
1066 1068 assert ip.user_ns["error"].at_eof()
1067 1069
1068 1070
1069 1071 async def test_script_bg_out_err():
1070 1072 ip = get_ipython()
1071 1073 ip.run_cell_magic(
1072 1074 "script",
1073 1075 f"--bg --out output --err error {sys.executable}",
1074 1076 "\n".join(
1075 1077 [
1076 1078 "import sys",
1077 1079 "print('hi')",
1078 1080 "print('hello', file=sys.stderr)",
1079 1081 ]
1080 1082 ),
1081 1083 )
1082 1084 assert (await ip.user_ns["output"].read()).strip() == b"hi"
1083 1085 assert (await ip.user_ns["error"].read()).strip() == b"hello"
1084 1086 assert ip.user_ns["output"].at_eof()
1085 1087 assert ip.user_ns["error"].at_eof()
1086 1088
1087 1089
1088 1090 async def test_script_bg_proc():
1089 1091 ip = get_ipython()
1090 1092 ip.run_cell_magic(
1091 1093 "script",
1092 1094 f"--bg --out output --proc p {sys.executable}",
1093 1095 "\n".join(
1094 1096 [
1095 1097 "import sys",
1096 1098 "print('hi')",
1097 1099 "print('hello', file=sys.stderr)",
1098 1100 ]
1099 1101 ),
1100 1102 )
1101 1103 p = ip.user_ns["p"]
1102 1104 await p.wait()
1103 1105 assert p.returncode == 0
1104 1106 assert (await p.stdout.read()).strip() == b"hi"
1105 1107 # not captured, so empty
1106 1108 assert (await p.stderr.read()) == b""
1107 1109 assert p.stdout.at_eof()
1108 1110 assert p.stderr.at_eof()
1109 1111
1110 1112
1111 1113 def test_script_defaults():
1112 1114 ip = get_ipython()
1113 1115 for cmd in ['sh', 'bash', 'perl', 'ruby']:
1114 1116 try:
1115 1117 find_cmd(cmd)
1116 1118 except Exception:
1117 1119 pass
1118 1120 else:
1119 1121 assert cmd in ip.magics_manager.magics["cell"]
1120 1122
1121 1123
1122 1124 @magics_class
1123 1125 class FooFoo(Magics):
1124 1126 """class with both %foo and %%foo magics"""
1125 1127 @line_magic('foo')
1126 1128 def line_foo(self, line):
1127 1129 "I am line foo"
1128 1130 pass
1129 1131
1130 1132 @cell_magic("foo")
1131 1133 def cell_foo(self, line, cell):
1132 1134 "I am cell foo, not line foo"
1133 1135 pass
1134 1136
1135 1137 def test_line_cell_info():
1136 1138 """%%foo and %foo magics are distinguishable to inspect"""
1137 1139 ip = get_ipython()
1138 1140 ip.magics_manager.register(FooFoo)
1139 1141 oinfo = ip.object_inspect("foo")
1140 1142 assert oinfo["found"] is True
1141 1143 assert oinfo["ismagic"] is True
1142 1144
1143 1145 oinfo = ip.object_inspect("%%foo")
1144 1146 assert oinfo["found"] is True
1145 1147 assert oinfo["ismagic"] is True
1146 1148 assert oinfo["docstring"] == FooFoo.cell_foo.__doc__
1147 1149
1148 1150 oinfo = ip.object_inspect("%foo")
1149 1151 assert oinfo["found"] is True
1150 1152 assert oinfo["ismagic"] is True
1151 1153 assert oinfo["docstring"] == FooFoo.line_foo.__doc__
1152 1154
1153 1155
1154 1156 def test_multiple_magics():
1155 1157 ip = get_ipython()
1156 1158 foo1 = FooFoo(ip)
1157 1159 foo2 = FooFoo(ip)
1158 1160 mm = ip.magics_manager
1159 1161 mm.register(foo1)
1160 1162 assert mm.magics["line"]["foo"].__self__ is foo1
1161 1163 mm.register(foo2)
1162 1164 assert mm.magics["line"]["foo"].__self__ is foo2
1163 1165
1164 1166
1165 1167 def test_alias_magic():
1166 1168 """Test %alias_magic."""
1167 1169 ip = get_ipython()
1168 1170 mm = ip.magics_manager
1169 1171
1170 1172 # Basic operation: both cell and line magics are created, if possible.
1171 1173 ip.run_line_magic("alias_magic", "timeit_alias timeit")
1172 1174 assert "timeit_alias" in mm.magics["line"]
1173 1175 assert "timeit_alias" in mm.magics["cell"]
1174 1176
1175 1177 # --cell is specified, line magic not created.
1176 1178 ip.run_line_magic("alias_magic", "--cell timeit_cell_alias timeit")
1177 1179 assert "timeit_cell_alias" not in mm.magics["line"]
1178 1180 assert "timeit_cell_alias" in mm.magics["cell"]
1179 1181
1180 1182 # Test that line alias is created successfully.
1181 1183 ip.run_line_magic("alias_magic", "--line env_alias env")
1182 1184 assert ip.run_line_magic("env", "") == ip.run_line_magic("env_alias", "")
1183 1185
1184 1186 # Test that line alias with parameters passed in is created successfully.
1185 1187 ip.run_line_magic(
1186 1188 "alias_magic", "--line history_alias history --params " + shlex.quote("3")
1187 1189 )
1188 1190 assert "history_alias" in mm.magics["line"]
1189 1191
1190 1192
1191 1193 def test_save():
1192 1194 """Test %save."""
1193 1195 ip = get_ipython()
1194 1196 ip.history_manager.reset() # Clear any existing history.
1195 1197 cmds = ["a=1", "def b():\n return a**2", "print(a, b())"]
1196 1198 for i, cmd in enumerate(cmds, start=1):
1197 1199 ip.history_manager.store_inputs(i, cmd)
1198 1200 with TemporaryDirectory() as tmpdir:
1199 1201 file = os.path.join(tmpdir, "testsave.py")
1200 1202 ip.run_line_magic("save", "%s 1-10" % file)
1201 1203 content = Path(file).read_text(encoding="utf-8")
1202 1204 assert content.count(cmds[0]) == 1
1203 1205 assert "coding: utf-8" in content
1204 1206 ip.run_line_magic("save", "-a %s 1-10" % file)
1205 1207 content = Path(file).read_text(encoding="utf-8")
1206 1208 assert content.count(cmds[0]) == 2
1207 1209 assert "coding: utf-8" in content
1208 1210
1209 1211
1210 1212 def test_save_with_no_args():
1211 1213 ip = get_ipython()
1212 1214 ip.history_manager.reset() # Clear any existing history.
1213 1215 cmds = ["a=1", "def b():\n return a**2", "print(a, b())", "%save"]
1214 1216 for i, cmd in enumerate(cmds, start=1):
1215 1217 ip.history_manager.store_inputs(i, cmd)
1216 1218
1217 1219 with TemporaryDirectory() as tmpdir:
1218 1220 path = os.path.join(tmpdir, "testsave.py")
1219 1221 ip.run_line_magic("save", path)
1220 1222 content = Path(path).read_text(encoding="utf-8")
1221 1223 expected_content = dedent(
1222 1224 """\
1223 1225 # coding: utf-8
1224 1226 a=1
1225 1227 def b():
1226 1228 return a**2
1227 1229 print(a, b())
1228 1230 """
1229 1231 )
1230 1232 assert content == expected_content
1231 1233
1232 1234
1233 1235 def test_store():
1234 1236 """Test %store."""
1235 1237 ip = get_ipython()
1236 1238 ip.run_line_magic('load_ext', 'storemagic')
1237 1239
1238 1240 # make sure the storage is empty
1239 1241 ip.run_line_magic("store", "-z")
1240 1242 ip.user_ns["var"] = 42
1241 1243 ip.run_line_magic("store", "var")
1242 1244 ip.user_ns["var"] = 39
1243 1245 ip.run_line_magic("store", "-r")
1244 1246 assert ip.user_ns["var"] == 42
1245 1247
1246 1248 ip.run_line_magic("store", "-d var")
1247 1249 ip.user_ns["var"] = 39
1248 1250 ip.run_line_magic("store", "-r")
1249 1251 assert ip.user_ns["var"] == 39
1250 1252
1251 1253
1252 1254 def _run_edit_test(arg_s, exp_filename=None,
1253 1255 exp_lineno=-1,
1254 1256 exp_contents=None,
1255 1257 exp_is_temp=None):
1256 1258 ip = get_ipython()
1257 1259 M = code.CodeMagics(ip)
1258 1260 last_call = ['','']
1259 1261 opts,args = M.parse_options(arg_s,'prxn:')
1260 1262 filename, lineno, is_temp = M._find_edit_target(ip, args, opts, last_call)
1261 1263
1262 1264 if exp_filename is not None:
1263 1265 assert exp_filename == filename
1264 1266 if exp_contents is not None:
1265 1267 with io.open(filename, 'r', encoding='utf-8') as f:
1266 1268 contents = f.read()
1267 1269 assert exp_contents == contents
1268 1270 if exp_lineno != -1:
1269 1271 assert exp_lineno == lineno
1270 1272 if exp_is_temp is not None:
1271 1273 assert exp_is_temp == is_temp
1272 1274
1273 1275
1274 1276 def test_edit_interactive():
1275 1277 """%edit on interactively defined objects"""
1276 1278 ip = get_ipython()
1277 1279 n = ip.execution_count
1278 1280 ip.run_cell("def foo(): return 1", store_history=True)
1279 1281
1280 1282 with pytest.raises(code.InteractivelyDefined) as e:
1281 1283 _run_edit_test("foo")
1282 1284 assert e.value.index == n
1283 1285
1284 1286
1285 1287 def test_edit_cell():
1286 1288 """%edit [cell id]"""
1287 1289 ip = get_ipython()
1288 1290
1289 1291 ip.run_cell("def foo(): return 1", store_history=True)
1290 1292
1291 1293 # test
1292 1294 _run_edit_test("1", exp_contents=ip.user_ns['In'][1], exp_is_temp=True)
1293 1295
1294 1296 def test_edit_fname():
1295 1297 """%edit file"""
1296 1298 # test
1297 1299 _run_edit_test("test file.py", exp_filename="test file.py")
1298 1300
1299 1301 def test_bookmark():
1300 1302 ip = get_ipython()
1301 1303 ip.run_line_magic('bookmark', 'bmname')
1302 1304 with tt.AssertPrints('bmname'):
1303 1305 ip.run_line_magic('bookmark', '-l')
1304 1306 ip.run_line_magic('bookmark', '-d bmname')
1305 1307
1306 1308 def test_ls_magic():
1307 1309 ip = get_ipython()
1308 1310 json_formatter = ip.display_formatter.formatters['application/json']
1309 1311 json_formatter.enabled = True
1310 1312 lsmagic = ip.magic('lsmagic')
1311 1313 with warnings.catch_warnings(record=True) as w:
1312 1314 j = json_formatter(lsmagic)
1313 1315 assert sorted(j) == ["cell", "line"]
1314 1316 assert w == [] # no warnings
1315 1317
1316 1318
1317 1319 def test_strip_initial_indent():
1318 1320 def sii(s):
1319 1321 lines = s.splitlines()
1320 1322 return '\n'.join(code.strip_initial_indent(lines))
1321 1323
1322 1324 assert sii(" a = 1\nb = 2") == "a = 1\nb = 2"
1323 1325 assert sii(" a\n b\nc") == "a\n b\nc"
1324 1326 assert sii("a\n b") == "a\n b"
1325 1327
1326 1328 def test_logging_magic_quiet_from_arg():
1327 1329 _ip.config.LoggingMagics.quiet = False
1328 1330 lm = logging.LoggingMagics(shell=_ip)
1329 1331 with TemporaryDirectory() as td:
1330 1332 try:
1331 1333 with tt.AssertNotPrints(re.compile("Activating.*")):
1332 1334 lm.logstart('-q {}'.format(
1333 1335 os.path.join(td, "quiet_from_arg.log")))
1334 1336 finally:
1335 1337 _ip.logger.logstop()
1336 1338
1337 1339 def test_logging_magic_quiet_from_config():
1338 1340 _ip.config.LoggingMagics.quiet = True
1339 1341 lm = logging.LoggingMagics(shell=_ip)
1340 1342 with TemporaryDirectory() as td:
1341 1343 try:
1342 1344 with tt.AssertNotPrints(re.compile("Activating.*")):
1343 1345 lm.logstart(os.path.join(td, "quiet_from_config.log"))
1344 1346 finally:
1345 1347 _ip.logger.logstop()
1346 1348
1347 1349
1348 1350 def test_logging_magic_not_quiet():
1349 1351 _ip.config.LoggingMagics.quiet = False
1350 1352 lm = logging.LoggingMagics(shell=_ip)
1351 1353 with TemporaryDirectory() as td:
1352 1354 try:
1353 1355 with tt.AssertPrints(re.compile("Activating.*")):
1354 1356 lm.logstart(os.path.join(td, "not_quiet.log"))
1355 1357 finally:
1356 1358 _ip.logger.logstop()
1357 1359
1358 1360
1359 1361 def test_time_no_var_expand():
1360 1362 _ip.user_ns['a'] = 5
1361 1363 _ip.user_ns['b'] = []
1362 1364 _ip.magic('time b.append("{a}")')
1363 1365 assert _ip.user_ns['b'] == ['{a}']
1364 1366
1365 1367
1366 1368 # this is slow, put at the end for local testing.
1367 1369 def test_timeit_arguments():
1368 1370 "Test valid timeit arguments, should not cause SyntaxError (GH #1269)"
1369 1371 _ip.magic("timeit -n1 -r1 a=('#')")
1370 1372
1371 1373
1372 1374 MINIMAL_LAZY_MAGIC = """
1373 1375 from IPython.core.magic import (
1374 1376 Magics,
1375 1377 magics_class,
1376 1378 line_magic,
1377 1379 cell_magic,
1378 1380 )
1379 1381
1380 1382
1381 1383 @magics_class
1382 1384 class LazyMagics(Magics):
1383 1385 @line_magic
1384 1386 def lazy_line(self, line):
1385 1387 print("Lazy Line")
1386 1388
1387 1389 @cell_magic
1388 1390 def lazy_cell(self, line, cell):
1389 1391 print("Lazy Cell")
1390 1392
1391 1393
1392 1394 def load_ipython_extension(ipython):
1393 1395 ipython.register_magics(LazyMagics)
1394 1396 """
1395 1397
1396 1398
1397 1399 def test_lazy_magics():
1398 1400 with pytest.raises(UsageError):
1399 1401 ip.run_line_magic("lazy_line", "")
1400 1402
1401 1403 startdir = os.getcwd()
1402 1404
1403 1405 with TemporaryDirectory() as tmpdir:
1404 1406 with prepended_to_syspath(tmpdir):
1405 1407 ptempdir = Path(tmpdir)
1406 1408 tf = ptempdir / "lazy_magic_module.py"
1407 1409 tf.write_text(MINIMAL_LAZY_MAGIC)
1408 1410 ip.magics_manager.register_lazy("lazy_line", Path(tf.name).name[:-3])
1409 1411 with tt.AssertPrints("Lazy Line"):
1410 1412 ip.run_line_magic("lazy_line", "")
1411 1413
1412 1414
1413 1415 TEST_MODULE = """
1414 1416 print('Loaded my_tmp')
1415 1417 if __name__ == "__main__":
1416 1418 print('I just ran a script')
1417 1419 """
1418 1420
1419 1421 def test_run_module_from_import_hook():
1420 1422 "Test that a module can be loaded via an import hook"
1421 1423 with TemporaryDirectory() as tmpdir:
1422 1424 fullpath = os.path.join(tmpdir, "my_tmp.py")
1423 1425 Path(fullpath).write_text(TEST_MODULE, encoding="utf-8")
1424 1426
1425 1427 import importlib.abc
1426 1428 import importlib.util
1427 1429
1428 1430 class MyTempImporter(importlib.abc.MetaPathFinder, importlib.abc.SourceLoader):
1429 1431 def find_spec(self, fullname, path, target=None):
1430 1432 if fullname == "my_tmp":
1431 1433 return importlib.util.spec_from_loader(fullname, self)
1432 1434
1433 1435 def get_filename(self, fullname):
1434 1436 assert fullname == "my_tmp"
1435 1437 return fullpath
1436 1438
1437 1439 def get_data(self, path):
1438 1440 assert Path(path).samefile(fullpath)
1439 1441 return Path(fullpath).read_text(encoding="utf-8")
1440 1442
1441 1443 sys.meta_path.insert(0, MyTempImporter())
1442 1444
1443 1445 with capture_output() as captured:
1444 1446 _ip.magic("run -m my_tmp")
1445 1447 _ip.run_cell("import my_tmp")
1446 1448
1447 1449 output = "Loaded my_tmp\nI just ran a script\nLoaded my_tmp\n"
1448 1450 assert output == captured.stdout
1449 1451
1450 1452 sys.meta_path.pop(0)
@@ -1,212 +1,216 b''
1 1 """Tests for various magic functions specific to the terminal frontend."""
2 2
3 3 #-----------------------------------------------------------------------------
4 4 # Imports
5 5 #-----------------------------------------------------------------------------
6 6
7 7 import sys
8 8 from io import StringIO
9 9 from unittest import TestCase
10 10
11 11 from IPython.testing import tools as tt
12 12 #-----------------------------------------------------------------------------
13 13 # Test functions begin
14 14 #-----------------------------------------------------------------------------
15 15
16 16
17 17 MINIMAL_LAZY_MAGIC = """
18 18 from IPython.core.magic import (
19 19 Magics,
20 20 magics_class,
21 21 line_magic,
22 22 cell_magic,
23 23 )
24 24
25 25
26 26 @magics_class
27 27 class LazyMagics(Magics):
28 28 @line_magic
29 29 def lazy_line(self, line):
30 30 print("Lazy Line")
31 31
32 32 @cell_magic
33 33 def lazy_cell(self, line, cell):
34 34 print("Lazy Cell")
35 35
36 36
37 37 def load_ipython_extension(ipython):
38 38 ipython.register_magics(LazyMagics)
39 39 """
40 40
41 41 def check_cpaste(code, should_fail=False):
42 42 """Execute code via 'cpaste' and ensure it was executed, unless
43 43 should_fail is set.
44 44 """
45 45 ip.user_ns['code_ran'] = False
46 46
47 47 src = StringIO()
48 48 src.write(code)
49 49 src.write('\n--\n')
50 50 src.seek(0)
51 51
52 52 stdin_save = sys.stdin
53 53 sys.stdin = src
54 54
55 55 try:
56 56 context = tt.AssertPrints if should_fail else tt.AssertNotPrints
57 57 with context("Traceback (most recent call last)"):
58 58 ip.run_line_magic("cpaste", "")
59 59
60 60 if not should_fail:
61 61 assert ip.user_ns['code_ran'], "%r failed" % code
62 62 finally:
63 63 sys.stdin = stdin_save
64 64
65 65 def test_cpaste():
66 66 """Test cpaste magic"""
67 67
68 68 def runf():
69 69 """Marker function: sets a flag when executed.
70 70 """
71 71 ip.user_ns['code_ran'] = True
72 72 return 'runf' # return string so '+ runf()' doesn't result in success
73 73
74 74 tests = {'pass': ["runf()",
75 75 "In [1]: runf()",
76 76 "In [1]: if 1:\n ...: runf()",
77 77 "> > > runf()",
78 78 ">>> runf()",
79 79 " >>> runf()",
80 80 ],
81 81
82 82 'fail': ["1 + runf()",
83 83 "++ runf()",
84 84 ]}
85 85
86 86 ip.user_ns['runf'] = runf
87 87
88 88 for code in tests['pass']:
89 89 check_cpaste(code)
90 90
91 91 for code in tests['fail']:
92 92 check_cpaste(code, should_fail=True)
93 93
94 94
95 95
96 96 class PasteTestCase(TestCase):
97 97 """Multiple tests for clipboard pasting"""
98 98
99 99 def paste(self, txt, flags='-q'):
100 100 """Paste input text, by default in quiet mode"""
101 101 ip.hooks.clipboard_get = lambda: txt
102 102 ip.run_line_magic("paste", flags)
103 103
104 104 def setUp(self):
105 105 # Inject fake clipboard hook but save original so we can restore it later
106 106 self.original_clip = ip.hooks.clipboard_get
107 107
108 108 def tearDown(self):
109 109 # Restore original hook
110 110 ip.hooks.clipboard_get = self.original_clip
111 111
112 112 def test_paste(self):
113 113 ip.user_ns.pop("x", None)
114 114 self.paste("x = 1")
115 115 self.assertEqual(ip.user_ns["x"], 1)
116 116 ip.user_ns.pop("x")
117 117
118 118 def test_paste_pyprompt(self):
119 119 ip.user_ns.pop("x", None)
120 120 self.paste(">>> x=2")
121 121 self.assertEqual(ip.user_ns["x"], 2)
122 122 ip.user_ns.pop("x")
123 123
124 124 def test_paste_py_multi(self):
125 self.paste("""
125 self.paste(
126 """
126 127 >>> x = [1,2,3]
127 128 >>> y = []
128 129 >>> for i in x:
129 130 ... y.append(i**2)
130 131 ...
131 132 """
132 133 )
133 134 self.assertEqual(ip.user_ns["x"], [1, 2, 3])
134 135 self.assertEqual(ip.user_ns["y"], [1, 4, 9])
135 136
136 137 def test_paste_py_multi_r(self):
137 138 "Now, test that self.paste -r works"
138 139 self.test_paste_py_multi()
139 140 self.assertEqual(ip.user_ns.pop("x"), [1, 2, 3])
140 141 self.assertEqual(ip.user_ns.pop("y"), [1, 4, 9])
141 142 self.assertFalse("x" in ip.user_ns)
142 143 ip.run_line_magic("paste", "-r")
143 144 self.assertEqual(ip.user_ns["x"], [1, 2, 3])
144 145 self.assertEqual(ip.user_ns["y"], [1, 4, 9])
145 146
146 147 def test_paste_email(self):
147 148 "Test pasting of email-quoted contents"
148 self.paste("""\
149 self.paste(
150 """\
149 151 >> def foo(x):
150 152 >> return x + 1
151 153 >> xx = foo(1.1)"""
152 154 )
153 155 self.assertEqual(ip.user_ns["xx"], 2.1)
154 156
155 157 def test_paste_email2(self):
156 158 "Email again; some programs add a space also at each quoting level"
157 self.paste("""\
159 self.paste(
160 """\
158 161 > > def foo(x):
159 162 > > return x + 1
160 163 > > yy = foo(2.1) """
161 164 )
162 165 self.assertEqual(ip.user_ns["yy"], 3.1)
163 166
164 167 def test_paste_email_py(self):
165 168 "Email quoting of interactive input"
166 self.paste("""\
169 self.paste(
170 """\
167 171 >> >>> def f(x):
168 172 >> ... return x+1
169 173 >> ...
170 174 >> >>> zz = f(2.5) """
171 175 )
172 176 self.assertEqual(ip.user_ns["zz"], 3.5)
173 177
174 178 def test_paste_echo(self):
175 179 "Also test self.paste echoing, by temporarily faking the writer"
176 180 w = StringIO()
177 181 old_write = sys.stdout.write
178 182 sys.stdout.write = w.write
179 183 code = """
180 184 a = 100
181 185 b = 200"""
182 186 try:
183 187 self.paste(code,'')
184 188 out = w.getvalue()
185 189 finally:
186 190 sys.stdout.write = old_write
187 191 self.assertEqual(ip.user_ns["a"], 100)
188 192 self.assertEqual(ip.user_ns["b"], 200)
189 193 assert out == code + "\n## -- End pasted text --\n"
190 194
191 195 def test_paste_leading_commas(self):
192 196 "Test multiline strings with leading commas"
193 197 tm = ip.magics_manager.registry['TerminalMagics']
194 198 s = '''\
195 199 a = """
196 200 ,1,2,3
197 201 """'''
198 202 ip.user_ns.pop("foo", None)
199 203 tm.store_or_execute(s, "foo")
200 204 self.assertIn("foo", ip.user_ns)
201 205
202 206 def test_paste_trailing_question(self):
203 207 "Test pasting sources with trailing question marks"
204 208 tm = ip.magics_manager.registry['TerminalMagics']
205 209 s = '''\
206 210 def funcfoo():
207 211 if True: #am i true?
208 212 return 'fooresult'
209 213 '''
210 214 ip.user_ns.pop('funcfoo', None)
211 215 self.paste(s)
212 216 self.assertEqual(ip.user_ns["funcfoo"](), "fooresult")
@@ -1,155 +1,155 b''
1 1 # coding: utf-8
2 2 """Tests for profile-related functions.
3 3
4 4 Currently only the startup-dir functionality is tested, but more tests should
5 5 be added for:
6 6
7 7 * ipython profile create
8 8 * ipython profile list
9 9 * ipython profile create --parallel
10 10 * security dir permissions
11 11
12 12 Authors
13 13 -------
14 14
15 15 * MinRK
16 16
17 17 """
18 18
19 19 #-----------------------------------------------------------------------------
20 20 # Imports
21 21 #-----------------------------------------------------------------------------
22 22
23 23 import shutil
24 24 import sys
25 25 import tempfile
26 26 from pathlib import Path
27 27 from unittest import TestCase
28 28
29 29 from tempfile import TemporaryDirectory
30 30
31 31 from IPython.core.profileapp import list_bundled_profiles, list_profiles_in
32 32 from IPython.core.profiledir import ProfileDir
33 33 from IPython.testing import decorators as dec
34 34 from IPython.testing import tools as tt
35 35 from IPython.utils.process import getoutput
36 36
37 37 #-----------------------------------------------------------------------------
38 38 # Globals
39 39 #-----------------------------------------------------------------------------
40 40 TMP_TEST_DIR = Path(tempfile.mkdtemp())
41 41 HOME_TEST_DIR = TMP_TEST_DIR / "home_test_dir"
42 42 IP_TEST_DIR = HOME_TEST_DIR / ".ipython"
43 43
44 44 #
45 45 # Setup/teardown functions/decorators
46 46 #
47 47
48 48 def setup_module():
49 49 """Setup test environment for the module:
50 50
51 51 - Adds dummy home dir tree
52 52 """
53 53 # Do not mask exceptions here. In particular, catching WindowsError is a
54 54 # problem because that exception is only defined on Windows...
55 55 (Path.cwd() / IP_TEST_DIR).mkdir(parents=True)
56 56
57 57
58 58 def teardown_module():
59 59 """Teardown test environment for the module:
60 60
61 61 - Remove dummy home dir tree
62 62 """
63 63 # Note: we remove the parent test dir, which is the root of all test
64 64 # subdirs we may have created. Use shutil instead of os.removedirs, so
65 65 # that non-empty directories are all recursively removed.
66 66 shutil.rmtree(TMP_TEST_DIR)
67 67
68 68
69 69 #-----------------------------------------------------------------------------
70 70 # Test functions
71 71 #-----------------------------------------------------------------------------
72 72 class ProfileStartupTest(TestCase):
73 73 def setUp(self):
74 74 # create profile dir
75 75 self.pd = ProfileDir.create_profile_dir_by_name(IP_TEST_DIR, "test")
76 76 self.options = ["--ipython-dir", IP_TEST_DIR, "--profile", "test"]
77 77 self.fname = TMP_TEST_DIR / "test.py"
78 78
79 79 def tearDown(self):
80 80 # We must remove this profile right away so its presence doesn't
81 81 # confuse other tests.
82 82 shutil.rmtree(self.pd.location)
83 83
84 84 def init(self, startup_file, startup, test):
85 85 # write startup python file
86 86 with open(Path(self.pd.startup_dir) / startup_file, "w", encoding="utf-8") as f:
87 87 f.write(startup)
88 88 # write simple test file, to check that the startup file was run
89 89 with open(self.fname, "w", encoding="utf-8") as f:
90 90 f.write(test)
91 91
92 92 def validate(self, output):
93 93 tt.ipexec_validate(self.fname, output, "", options=self.options)
94 94
95 95 def test_startup_py(self):
96 96 self.init('00-start.py', 'zzz=123\n', 'print(zzz)\n')
97 97 self.validate('123')
98 98
99 99 def test_startup_ipy(self):
100 100 self.init('00-start.ipy', '%xmode plain\n', '')
101 101 self.validate('Exception reporting mode: Plain')
102 102
103 103
104 104 def test_list_profiles_in():
105 105 # No need to remove these directories and files, as they will get nuked in
106 106 # the module-level teardown.
107 107 td = Path(tempfile.mkdtemp(dir=TMP_TEST_DIR))
108 108 for name in ("profile_foo", "profile_hello", "not_a_profile"):
109 109 Path(td / name).mkdir(parents=True)
110 110 if dec.unicode_paths:
111 Path(td / u"profile_ünicode").mkdir(parents=True)
111 Path(td / "profile_ünicode").mkdir(parents=True)
112 112
113 113 with open(td / "profile_file", "w", encoding="utf-8") as f:
114 114 f.write("I am not a profile directory")
115 115 profiles = list_profiles_in(td)
116 116
117 117 # unicode normalization can turn u'ünicode' into u'u\0308nicode',
118 118 # so only check for *nicode, and that creating a ProfileDir from the
119 119 # name remains valid
120 120 found_unicode = False
121 121 for p in list(profiles):
122 122 if p.endswith('nicode'):
123 123 pd = ProfileDir.find_profile_dir_by_name(td, p)
124 124 profiles.remove(p)
125 125 found_unicode = True
126 126 break
127 127 if dec.unicode_paths:
128 128 assert found_unicode is True
129 129 assert set(profiles) == {"foo", "hello"}
130 130
131 131
132 132 def test_list_bundled_profiles():
133 133 # This variable will need to be updated when a new profile gets bundled
134 134 bundled = sorted(list_bundled_profiles())
135 135 assert bundled == []
136 136
137 137
138 138 def test_profile_create_ipython_dir():
139 139 """ipython profile create respects --ipython-dir"""
140 140 with TemporaryDirectory() as td:
141 141 getoutput(
142 142 [
143 143 sys.executable,
144 144 "-m",
145 145 "IPython",
146 146 "profile",
147 147 "create",
148 148 "foo",
149 149 "--ipython-dir=%s" % td,
150 150 ]
151 151 )
152 152 profile_dir = Path(td) / "profile_foo"
153 153 assert Path(profile_dir).exists()
154 154 ipython_config = profile_dir / "ipython_config.py"
155 155 assert Path(ipython_config).exists()
@@ -1,397 +1,399 b''
1 1 """
2 2 This module contains factory functions that attempt
3 3 to return Qt submodules from the various python Qt bindings.
4 4
5 5 It also protects against double-importing Qt with different
6 6 bindings, which is unstable and likely to crash
7 7
8 8 This is used primarily by qt and qt_for_kernel, and shouldn't
9 9 be accessed directly from the outside
10 10 """
11 11 import importlib.abc
12 12 import sys
13 13 import types
14 14 from functools import partial, lru_cache
15 15 import operator
16 16
17 17 # ### Available APIs.
18 18 # Qt6
19 19 QT_API_PYQT6 = "pyqt6"
20 20 QT_API_PYSIDE6 = "pyside6"
21 21
22 22 # Qt5
23 23 QT_API_PYQT5 = 'pyqt5'
24 24 QT_API_PYSIDE2 = 'pyside2'
25 25
26 26 # Qt4
27 27 QT_API_PYQT = "pyqt" # Force version 2
28 28 QT_API_PYQTv1 = "pyqtv1" # Force version 2
29 29 QT_API_PYSIDE = "pyside"
30 30
31 31 QT_API_PYQT_DEFAULT = "pyqtdefault" # use system default for version 1 vs. 2
32 32
33 33 api_to_module = {
34 34 # Qt6
35 35 QT_API_PYQT6: "PyQt6",
36 36 QT_API_PYSIDE6: "PySide6",
37 37 # Qt5
38 38 QT_API_PYQT5: "PyQt5",
39 39 QT_API_PYSIDE2: "PySide2",
40 40 # Qt4
41 41 QT_API_PYSIDE: "PySide",
42 42 QT_API_PYQT: "PyQt4",
43 43 QT_API_PYQTv1: "PyQt4",
44 44 # default
45 45 QT_API_PYQT_DEFAULT: "PyQt6",
46 46 }
47 47
48 48
49 49 class ImportDenier(importlib.abc.MetaPathFinder):
50 50 """Import Hook that will guard against bad Qt imports
51 51 once IPython commits to a specific binding
52 52 """
53 53
54 54 def __init__(self):
55 55 self.__forbidden = set()
56 56
57 57 def forbid(self, module_name):
58 58 sys.modules.pop(module_name, None)
59 59 self.__forbidden.add(module_name)
60 60
61 61 def find_spec(self, fullname, path, target=None):
62 62 if path:
63 63 return
64 64 if fullname in self.__forbidden:
65 65 raise ImportError(
66 66 """
67 67 Importing %s disabled by IPython, which has
68 68 already imported an Incompatible QT Binding: %s
69 """ % (fullname, loaded_api()))
69 """
70 % (fullname, loaded_api())
71 )
70 72
71 73
72 74 ID = ImportDenier()
73 75 sys.meta_path.insert(0, ID)
74 76
75 77
76 78 def commit_api(api):
77 79 """Commit to a particular API, and trigger ImportErrors on subsequent
78 80 dangerous imports"""
79 81 modules = set(api_to_module.values())
80 82
81 83 modules.remove(api_to_module[api])
82 84 for mod in modules:
83 85 ID.forbid(mod)
84 86
85 87
86 88 def loaded_api():
87 89 """Return which API is loaded, if any
88 90
89 91 If this returns anything besides None,
90 92 importing any other Qt binding is unsafe.
91 93
92 94 Returns
93 95 -------
94 96 None, 'pyside6', 'pyqt6', 'pyside2', 'pyside', 'pyqt', 'pyqt5', 'pyqtv1'
95 97 """
96 98 if sys.modules.get("PyQt6.QtCore"):
97 99 return QT_API_PYQT6
98 100 elif sys.modules.get("PySide6.QtCore"):
99 101 return QT_API_PYSIDE6
100 102 elif sys.modules.get("PyQt5.QtCore"):
101 103 return QT_API_PYQT5
102 104 elif sys.modules.get("PySide2.QtCore"):
103 105 return QT_API_PYSIDE2
104 106 elif sys.modules.get("PyQt4.QtCore"):
105 107 if qtapi_version() == 2:
106 108 return QT_API_PYQT
107 109 else:
108 110 return QT_API_PYQTv1
109 111 elif sys.modules.get("PySide.QtCore"):
110 112 return QT_API_PYSIDE
111 113
112 114 return None
113 115
114 116
115 117 def has_binding(api):
116 118 """Safely check for PyQt4/5, PySide or PySide2, without importing submodules
117 119
118 120 Parameters
119 121 ----------
120 122 api : str [ 'pyqtv1' | 'pyqt' | 'pyqt5' | 'pyside' | 'pyside2' | 'pyqtdefault']
121 123 Which module to check for
122 124
123 125 Returns
124 126 -------
125 127 True if the relevant module appears to be importable
126 128 """
127 129 module_name = api_to_module[api]
128 130 from importlib.util import find_spec
129 131
130 132 required = ['QtCore', 'QtGui', 'QtSvg']
131 133 if api in (QT_API_PYQT5, QT_API_PYSIDE2, QT_API_PYQT6, QT_API_PYSIDE6):
132 134 # QT5 requires QtWidgets too
133 135 required.append('QtWidgets')
134 136
135 137 for submod in required:
136 138 try:
137 139 spec = find_spec('%s.%s' % (module_name, submod))
138 140 except ImportError:
139 141 # Package (e.g. PyQt5) not found
140 142 return False
141 143 else:
142 144 if spec is None:
143 145 # Submodule (e.g. PyQt5.QtCore) not found
144 146 return False
145 147
146 148 if api == QT_API_PYSIDE:
147 149 # We can also safely check PySide version
148 150 import PySide
149 151
150 152 return PySide.__version_info__ >= (1, 0, 3)
151 153
152 154 return True
153 155
154 156
155 157 def qtapi_version():
156 158 """Return which QString API has been set, if any
157 159
158 160 Returns
159 161 -------
160 162 The QString API version (1 or 2), or None if not set
161 163 """
162 164 try:
163 165 import sip
164 166 except ImportError:
165 167 # as of PyQt5 5.11, sip is no longer available as a top-level
166 168 # module and needs to be imported from the PyQt5 namespace
167 169 try:
168 170 from PyQt5 import sip
169 171 except ImportError:
170 172 return
171 173 try:
172 174 return sip.getapi('QString')
173 175 except ValueError:
174 176 return
175 177
176 178
177 179 def can_import(api):
178 180 """Safely query whether an API is importable, without importing it"""
179 181 if not has_binding(api):
180 182 return False
181 183
182 184 current = loaded_api()
183 185 if api == QT_API_PYQT_DEFAULT:
184 186 return current in [QT_API_PYQT6, None]
185 187 else:
186 188 return current in [api, None]
187 189
188 190
189 191 def import_pyqt4(version=2):
190 192 """
191 193 Import PyQt4
192 194
193 195 Parameters
194 196 ----------
195 197 version : 1, 2, or None
196 198 Which QString/QVariant API to use. Set to None to use the system
197 199 default
198 200 ImportErrors raised within this function are non-recoverable
199 201 """
200 202 # The new-style string API (version=2) automatically
201 203 # converts QStrings to Unicode Python strings. Also, automatically unpacks
202 204 # QVariants to their underlying objects.
203 205 import sip
204 206
205 207 if version is not None:
206 208 sip.setapi('QString', version)
207 209 sip.setapi('QVariant', version)
208 210
209 211 from PyQt4 import QtGui, QtCore, QtSvg
210 212
211 213 if QtCore.PYQT_VERSION < 0x040700:
212 214 raise ImportError("IPython requires PyQt4 >= 4.7, found %s" %
213 215 QtCore.PYQT_VERSION_STR)
214 216
215 217 # Alias PyQt-specific functions for PySide compatibility.
216 218 QtCore.Signal = QtCore.pyqtSignal
217 219 QtCore.Slot = QtCore.pyqtSlot
218 220
219 221 # query for the API version (in case version == None)
220 222 version = sip.getapi('QString')
221 223 api = QT_API_PYQTv1 if version == 1 else QT_API_PYQT
222 224 return QtCore, QtGui, QtSvg, api
223 225
224 226
225 227 def import_pyqt5():
226 228 """
227 229 Import PyQt5
228 230
229 231 ImportErrors raised within this function are non-recoverable
230 232 """
231 233
232 234 from PyQt5 import QtCore, QtSvg, QtWidgets, QtGui
233 235
234 236 # Alias PyQt-specific functions for PySide compatibility.
235 237 QtCore.Signal = QtCore.pyqtSignal
236 238 QtCore.Slot = QtCore.pyqtSlot
237 239
238 240 # Join QtGui and QtWidgets for Qt4 compatibility.
239 241 QtGuiCompat = types.ModuleType('QtGuiCompat')
240 242 QtGuiCompat.__dict__.update(QtGui.__dict__)
241 243 QtGuiCompat.__dict__.update(QtWidgets.__dict__)
242 244
243 245 api = QT_API_PYQT5
244 246 return QtCore, QtGuiCompat, QtSvg, api
245 247
246 248
247 249 def import_pyqt6():
248 250 """
249 251 Import PyQt6
250 252
251 253 ImportErrors raised within this function are non-recoverable
252 254 """
253 255
254 256 from PyQt6 import QtCore, QtSvg, QtWidgets, QtGui
255 257
256 258 # Alias PyQt-specific functions for PySide compatibility.
257 259 QtCore.Signal = QtCore.pyqtSignal
258 260 QtCore.Slot = QtCore.pyqtSlot
259 261
260 262 # Join QtGui and QtWidgets for Qt4 compatibility.
261 263 QtGuiCompat = types.ModuleType("QtGuiCompat")
262 264 QtGuiCompat.__dict__.update(QtGui.__dict__)
263 265 QtGuiCompat.__dict__.update(QtWidgets.__dict__)
264 266
265 267 api = QT_API_PYQT6
266 268 return QtCore, QtGuiCompat, QtSvg, api
267 269
268 270
269 271 def import_pyside():
270 272 """
271 273 Import PySide
272 274
273 275 ImportErrors raised within this function are non-recoverable
274 276 """
275 277 from PySide import QtGui, QtCore, QtSvg
276 278 return QtCore, QtGui, QtSvg, QT_API_PYSIDE
277 279
278 280 def import_pyside2():
279 281 """
280 282 Import PySide2
281 283
282 284 ImportErrors raised within this function are non-recoverable
283 285 """
284 286 from PySide2 import QtGui, QtCore, QtSvg, QtWidgets, QtPrintSupport
285 287
286 288 # Join QtGui and QtWidgets for Qt4 compatibility.
287 289 QtGuiCompat = types.ModuleType('QtGuiCompat')
288 290 QtGuiCompat.__dict__.update(QtGui.__dict__)
289 291 QtGuiCompat.__dict__.update(QtWidgets.__dict__)
290 292 QtGuiCompat.__dict__.update(QtPrintSupport.__dict__)
291 293
292 294 return QtCore, QtGuiCompat, QtSvg, QT_API_PYSIDE2
293 295
294 296
295 297 def import_pyside6():
296 298 """
297 299 Import PySide6
298 300
299 301 ImportErrors raised within this function are non-recoverable
300 302 """
301 303 from PySide6 import QtGui, QtCore, QtSvg, QtWidgets, QtPrintSupport
302 304
303 305 # Join QtGui and QtWidgets for Qt4 compatibility.
304 306 QtGuiCompat = types.ModuleType("QtGuiCompat")
305 307 QtGuiCompat.__dict__.update(QtGui.__dict__)
306 308 QtGuiCompat.__dict__.update(QtWidgets.__dict__)
307 309 QtGuiCompat.__dict__.update(QtPrintSupport.__dict__)
308 310
309 311 return QtCore, QtGuiCompat, QtSvg, QT_API_PYSIDE6
310 312
311 313
312 314 def load_qt(api_options):
313 315 """
314 316 Attempt to import Qt, given a preference list
315 317 of permissible bindings
316 318
317 319 It is safe to call this function multiple times.
318 320
319 321 Parameters
320 322 ----------
321 323 api_options : List of strings
322 324 The order of APIs to try. Valid items are 'pyside', 'pyside2',
323 325 'pyqt', 'pyqt5', 'pyqtv1' and 'pyqtdefault'
324 326
325 327 Returns
326 328 -------
327 329 A tuple of QtCore, QtGui, QtSvg, QT_API
328 330 The first three are the Qt modules. The last is the
329 331 string indicating which module was loaded.
330 332
331 333 Raises
332 334 ------
333 335 ImportError, if it isn't possible to import any requested
334 336 bindings (either because they aren't installed, or because
335 337 an incompatible library has already been installed)
336 338 """
337 339 loaders = {
338 340 # Qt6
339 341 QT_API_PYQT6: import_pyqt6,
340 342 QT_API_PYSIDE6: import_pyside6,
341 343 # Qt5
342 344 QT_API_PYQT5: import_pyqt5,
343 345 QT_API_PYSIDE2: import_pyside2,
344 346 # Qt4
345 347 QT_API_PYSIDE: import_pyside,
346 348 QT_API_PYQT: import_pyqt4,
347 349 QT_API_PYQTv1: partial(import_pyqt4, version=1),
348 350 # default
349 351 QT_API_PYQT_DEFAULT: import_pyqt6,
350 352 }
351 353
352 354 for api in api_options:
353 355
354 356 if api not in loaders:
355 357 raise RuntimeError(
356 358 "Invalid Qt API %r, valid values are: %s" %
357 359 (api, ", ".join(["%r" % k for k in loaders.keys()])))
358 360
359 361 if not can_import(api):
360 362 continue
361 363
362 364 #cannot safely recover from an ImportError during this
363 365 result = loaders[api]()
364 366 api = result[-1] # changed if api = QT_API_PYQT_DEFAULT
365 367 commit_api(api)
366 368 return result
367 369 else:
368 370 raise ImportError("""
369 371 Could not load requested Qt binding. Please ensure that
370 372 PyQt4 >= 4.7, PyQt5, PySide >= 1.0.3 or PySide2 is available,
371 373 and only one is imported per session.
372 374
373 375 Currently-imported Qt library: %r
374 376 PyQt4 available (requires QtCore, QtGui, QtSvg): %s
375 377 PyQt5 available (requires QtCore, QtGui, QtSvg, QtWidgets): %s
376 378 PySide >= 1.0.3 installed: %s
377 379 PySide2 installed: %s
378 380 Tried to load: %r
379 381 """ % (loaded_api(),
380 382 has_binding(QT_API_PYQT),
381 383 has_binding(QT_API_PYQT5),
382 384 has_binding(QT_API_PYSIDE),
383 385 has_binding(QT_API_PYSIDE2),
384 386 api_options))
385 387
386 388
387 389 def enum_factory(QT_API, QtCore):
388 390 """Construct an enum helper to account for PyQt5 <-> PyQt6 changes."""
389 391
390 392 @lru_cache(None)
391 393 def _enum(name):
392 394 # foo.bar.Enum.Entry (PyQt6) <=> foo.bar.Entry (non-PyQt6).
393 395 return operator.attrgetter(
394 396 name if QT_API == QT_API_PYQT6 else name.rpartition(".")[0]
395 397 )(sys.modules[QtCore.__package__])
396 398
397 399 return _enum
@@ -1,677 +1,675 b''
1 1 """Various display related classes.
2 2
3 3 Authors : MinRK, gregcaporaso, dannystaple
4 4 """
5 5 from html import escape as html_escape
6 6 from os.path import exists, isfile, splitext, abspath, join, isdir
7 7 from os import walk, sep, fsdecode
8 8
9 9 from IPython.core.display import DisplayObject, TextDisplayObject
10 10
11 11 from typing import Tuple, Iterable
12 12
13 13 __all__ = ['Audio', 'IFrame', 'YouTubeVideo', 'VimeoVideo', 'ScribdDocument',
14 14 'FileLink', 'FileLinks', 'Code']
15 15
16 16
17 17 class Audio(DisplayObject):
18 18 """Create an audio object.
19 19
20 20 When this object is returned by an input cell or passed to the
21 21 display function, it will result in Audio controls being displayed
22 22 in the frontend (only works in the notebook).
23 23
24 24 Parameters
25 25 ----------
26 26 data : numpy array, list, unicode, str or bytes
27 27 Can be one of
28 28
29 29 * Numpy 1d array containing the desired waveform (mono)
30 30 * Numpy 2d array containing waveforms for each channel.
31 31 Shape=(NCHAN, NSAMPLES). For the standard channel order, see
32 32 http://msdn.microsoft.com/en-us/library/windows/hardware/dn653308(v=vs.85).aspx
33 33 * List of float or integer representing the waveform (mono)
34 34 * String containing the filename
35 35 * Bytestring containing raw PCM data or
36 36 * URL pointing to a file on the web.
37 37
38 38 If the array option is used, the waveform will be normalized.
39 39
40 40 If a filename or url is used, the format support will be browser
41 41 dependent.
42 42 url : unicode
43 43 A URL to download the data from.
44 44 filename : unicode
45 45 Path to a local file to load the data from.
46 46 embed : boolean
47 47 Should the audio data be embedded using a data URI (True) or should
48 48 the original source be referenced. Set this to True if you want the
49 49 audio to playable later with no internet connection in the notebook.
50 50
51 51 Default is `True`, unless the keyword argument `url` is set, then
52 52 default value is `False`.
53 53 rate : integer
54 54 The sampling rate of the raw data.
55 55 Only required when data parameter is being used as an array
56 56 autoplay : bool
57 57 Set to True if the audio should immediately start playing.
58 58 Default is `False`.
59 59 normalize : bool
60 60 Whether audio should be normalized (rescaled) to the maximum possible
61 61 range. Default is `True`. When set to `False`, `data` must be between
62 62 -1 and 1 (inclusive), otherwise an error is raised.
63 63 Applies only when `data` is a list or array of samples; other types of
64 64 audio are never normalized.
65 65
66 66 Examples
67 67 --------
68 68
69 69 >>> import pytest
70 70 >>> np = pytest.importorskip("numpy")
71 71
72 72 Generate a sound
73 73
74 74 >>> import numpy as np
75 75 >>> framerate = 44100
76 76 >>> t = np.linspace(0,5,framerate*5)
77 77 >>> data = np.sin(2*np.pi*220*t) + np.sin(2*np.pi*224*t)
78 78 >>> Audio(data, rate=framerate)
79 79 <IPython.lib.display.Audio object>
80 80
81 81 Can also do stereo or more channels
82 82
83 83 >>> dataleft = np.sin(2*np.pi*220*t)
84 84 >>> dataright = np.sin(2*np.pi*224*t)
85 85 >>> Audio([dataleft, dataright], rate=framerate)
86 86 <IPython.lib.display.Audio object>
87 87
88 88 From URL:
89 89
90 90 >>> Audio("http://www.nch.com.au/acm/8k16bitpcm.wav") # doctest: +SKIP
91 91 >>> Audio(url="http://www.w3schools.com/html/horse.ogg") # doctest: +SKIP
92 92
93 93 From a File:
94 94
95 95 >>> Audio('IPython/lib/tests/test.wav') # doctest: +SKIP
96 96 >>> Audio(filename='IPython/lib/tests/test.wav') # doctest: +SKIP
97 97
98 98 From Bytes:
99 99
100 100 >>> Audio(b'RAW_WAV_DATA..') # doctest: +SKIP
101 101 >>> Audio(data=b'RAW_WAV_DATA..') # doctest: +SKIP
102 102
103 103 See Also
104 104 --------
105 105 ipywidgets.Audio
106
107 AUdio widget with more more flexibility and options.
108
106
107 Audio widget with more more flexibility and options.
108
109 109 """
110 110 _read_flags = 'rb'
111 111
112 112 def __init__(self, data=None, filename=None, url=None, embed=None, rate=None, autoplay=False, normalize=True, *,
113 113 element_id=None):
114 114 if filename is None and url is None and data is None:
115 115 raise ValueError("No audio data found. Expecting filename, url, or data.")
116 116 if embed is False and url is None:
117 117 raise ValueError("No url found. Expecting url when embed=False")
118 118
119 119 if url is not None and embed is not True:
120 120 self.embed = False
121 121 else:
122 122 self.embed = True
123 123 self.autoplay = autoplay
124 124 self.element_id = element_id
125 125 super(Audio, self).__init__(data=data, url=url, filename=filename)
126 126
127 127 if self.data is not None and not isinstance(self.data, bytes):
128 128 if rate is None:
129 129 raise ValueError("rate must be specified when data is a numpy array or list of audio samples.")
130 130 self.data = Audio._make_wav(data, rate, normalize)
131 131
132 132 def reload(self):
133 133 """Reload the raw data from file or URL."""
134 134 import mimetypes
135 135 if self.embed:
136 136 super(Audio, self).reload()
137 137
138 138 if self.filename is not None:
139 139 self.mimetype = mimetypes.guess_type(self.filename)[0]
140 140 elif self.url is not None:
141 141 self.mimetype = mimetypes.guess_type(self.url)[0]
142 142 else:
143 143 self.mimetype = "audio/wav"
144 144
145 145 @staticmethod
146 146 def _make_wav(data, rate, normalize):
147 147 """ Transform a numpy array to a PCM bytestring """
148 148 from io import BytesIO
149 149 import wave
150 150
151 151 try:
152 152 scaled, nchan = Audio._validate_and_normalize_with_numpy(data, normalize)
153 153 except ImportError:
154 154 scaled, nchan = Audio._validate_and_normalize_without_numpy(data, normalize)
155 155
156 156 fp = BytesIO()
157 157 waveobj = wave.open(fp,mode='wb')
158 158 waveobj.setnchannels(nchan)
159 159 waveobj.setframerate(rate)
160 160 waveobj.setsampwidth(2)
161 161 waveobj.setcomptype('NONE','NONE')
162 162 waveobj.writeframes(scaled)
163 163 val = fp.getvalue()
164 164 waveobj.close()
165 165
166 166 return val
167 167
168 168 @staticmethod
169 169 def _validate_and_normalize_with_numpy(data, normalize) -> Tuple[bytes, int]:
170 170 import numpy as np
171 171
172 172 data = np.array(data, dtype=float)
173 173 if len(data.shape) == 1:
174 174 nchan = 1
175 175 elif len(data.shape) == 2:
176 176 # In wave files,channels are interleaved. E.g.,
177 177 # "L1R1L2R2..." for stereo. See
178 178 # http://msdn.microsoft.com/en-us/library/windows/hardware/dn653308(v=vs.85).aspx
179 179 # for channel ordering
180 180 nchan = data.shape[0]
181 181 data = data.T.ravel()
182 182 else:
183 183 raise ValueError('Array audio input must be a 1D or 2D array')
184 184
185 185 max_abs_value = np.max(np.abs(data))
186 186 normalization_factor = Audio._get_normalization_factor(max_abs_value, normalize)
187 187 scaled = data / normalization_factor * 32767
188 188 return scaled.astype("<h").tobytes(), nchan
189 189
190 190 @staticmethod
191 191 def _validate_and_normalize_without_numpy(data, normalize):
192 192 import array
193 193 import sys
194 194
195 195 data = array.array('f', data)
196 196
197 197 try:
198 198 max_abs_value = float(max([abs(x) for x in data]))
199 199 except TypeError as e:
200 200 raise TypeError('Only lists of mono audio are '
201 201 'supported if numpy is not installed') from e
202 202
203 203 normalization_factor = Audio._get_normalization_factor(max_abs_value, normalize)
204 204 scaled = array.array('h', [int(x / normalization_factor * 32767) for x in data])
205 205 if sys.byteorder == 'big':
206 206 scaled.byteswap()
207 207 nchan = 1
208 208 return scaled.tobytes(), nchan
209 209
210 210 @staticmethod
211 211 def _get_normalization_factor(max_abs_value, normalize):
212 212 if not normalize and max_abs_value > 1:
213 213 raise ValueError('Audio data must be between -1 and 1 when normalize=False.')
214 214 return max_abs_value if normalize else 1
215 215
216 216 def _data_and_metadata(self):
217 217 """shortcut for returning metadata with url information, if defined"""
218 218 md = {}
219 219 if self.url:
220 220 md['url'] = self.url
221 221 if md:
222 222 return self.data, md
223 223 else:
224 224 return self.data
225 225
226 226 def _repr_html_(self):
227 227 src = """
228 228 <audio {element_id} controls="controls" {autoplay}>
229 229 <source src="{src}" type="{type}" />
230 230 Your browser does not support the audio element.
231 231 </audio>
232 232 """
233 233 return src.format(src=self.src_attr(), type=self.mimetype, autoplay=self.autoplay_attr(),
234 234 element_id=self.element_id_attr())
235 235
236 236 def src_attr(self):
237 237 import base64
238 238 if self.embed and (self.data is not None):
239 239 data = base64=base64.b64encode(self.data).decode('ascii')
240 240 return """data:{type};base64,{base64}""".format(type=self.mimetype,
241 241 base64=data)
242 242 elif self.url is not None:
243 243 return self.url
244 244 else:
245 245 return ""
246 246
247 247 def autoplay_attr(self):
248 248 if(self.autoplay):
249 249 return 'autoplay="autoplay"'
250 250 else:
251 251 return ''
252 252
253 253 def element_id_attr(self):
254 254 if (self.element_id):
255 255 return 'id="{element_id}"'.format(element_id=self.element_id)
256 256 else:
257 257 return ''
258 258
259 259 class IFrame(object):
260 260 """
261 261 Generic class to embed an iframe in an IPython notebook
262 262 """
263 263
264 264 iframe = """
265 265 <iframe
266 266 width="{width}"
267 267 height="{height}"
268 268 src="{src}{params}"
269 269 frameborder="0"
270 270 allowfullscreen
271 271 {extras}
272 272 ></iframe>
273 273 """
274 274
275 275 def __init__(self, src, width, height, extras: Iterable[str] = None, **kwargs):
276 276 if extras is None:
277 277 extras = []
278 278
279 279 self.src = src
280 280 self.width = width
281 281 self.height = height
282 282 self.extras = extras
283 283 self.params = kwargs
284 284
285 285 def _repr_html_(self):
286 286 """return the embed iframe"""
287 287 if self.params:
288 288 from urllib.parse import urlencode
289 289 params = "?" + urlencode(self.params)
290 290 else:
291 291 params = ""
292 292 return self.iframe.format(
293 293 src=self.src,
294 294 width=self.width,
295 295 height=self.height,
296 296 params=params,
297 297 extras=" ".join(self.extras),
298 298 )
299 299
300 300
301 301 class YouTubeVideo(IFrame):
302 302 """Class for embedding a YouTube Video in an IPython session, based on its video id.
303 303
304 304 e.g. to embed the video from https://www.youtube.com/watch?v=foo , you would
305 305 do::
306 306
307 307 vid = YouTubeVideo("foo")
308 308 display(vid)
309 309
310 310 To start from 30 seconds::
311 311
312 312 vid = YouTubeVideo("abc", start=30)
313 313 display(vid)
314 314
315 315 To calculate seconds from time as hours, minutes, seconds use
316 316 :class:`datetime.timedelta`::
317 317
318 318 start=int(timedelta(hours=1, minutes=46, seconds=40).total_seconds())
319 319
320 320 Other parameters can be provided as documented at
321 321 https://developers.google.com/youtube/player_parameters#Parameters
322 322
323 323 When converting the notebook using nbconvert, a jpeg representation of the video
324 324 will be inserted in the document.
325 325 """
326 326
327 327 def __init__(self, id, width=400, height=300, allow_autoplay=False, **kwargs):
328 328 self.id=id
329 329 src = "https://www.youtube.com/embed/{0}".format(id)
330 330 if allow_autoplay:
331 331 extras = list(kwargs.get("extras", [])) + ['allow="autoplay"']
332 332 kwargs.update(autoplay=1, extras=extras)
333 333 super(YouTubeVideo, self).__init__(src, width, height, **kwargs)
334 334
335 335 def _repr_jpeg_(self):
336 336 # Deferred import
337 337 from urllib.request import urlopen
338 338
339 339 try:
340 340 return urlopen("https://img.youtube.com/vi/{id}/hqdefault.jpg".format(id=self.id)).read()
341 341 except IOError:
342 342 return None
343 343
344 344 class VimeoVideo(IFrame):
345 345 """
346 346 Class for embedding a Vimeo video in an IPython session, based on its video id.
347 347 """
348 348
349 349 def __init__(self, id, width=400, height=300, **kwargs):
350 350 src="https://player.vimeo.com/video/{0}".format(id)
351 351 super(VimeoVideo, self).__init__(src, width, height, **kwargs)
352 352
353 353 class ScribdDocument(IFrame):
354 354 """
355 355 Class for embedding a Scribd document in an IPython session
356 356
357 357 Use the start_page params to specify a starting point in the document
358 358 Use the view_mode params to specify display type one off scroll | slideshow | book
359 359
360 360 e.g to Display Wes' foundational paper about PANDAS in book mode from page 3
361 361
362 362 ScribdDocument(71048089, width=800, height=400, start_page=3, view_mode="book")
363 363 """
364 364
365 365 def __init__(self, id, width=400, height=300, **kwargs):
366 366 src="https://www.scribd.com/embeds/{0}/content".format(id)
367 367 super(ScribdDocument, self).__init__(src, width, height, **kwargs)
368 368
369 369 class FileLink(object):
370 370 """Class for embedding a local file link in an IPython session, based on path
371 371
372 372 e.g. to embed a link that was generated in the IPython notebook as my/data.txt
373 373
374 374 you would do::
375 375
376 376 local_file = FileLink("my/data.txt")
377 377 display(local_file)
378 378
379 379 or in the HTML notebook, just::
380 380
381 381 FileLink("my/data.txt")
382 382 """
383 383
384 384 html_link_str = "<a href='%s' target='_blank'>%s</a>"
385 385
386 386 def __init__(self,
387 387 path,
388 388 url_prefix='',
389 389 result_html_prefix='',
390 390 result_html_suffix='<br>'):
391 391 """
392 392 Parameters
393 393 ----------
394 394 path : str
395 395 path to the file or directory that should be formatted
396 396 url_prefix : str
397 397 prefix to be prepended to all files to form a working link [default:
398 398 '']
399 399 result_html_prefix : str
400 400 text to append to beginning to link [default: '']
401 401 result_html_suffix : str
402 402 text to append at the end of link [default: '<br>']
403 403 """
404 404 if isdir(path):
405 405 raise ValueError("Cannot display a directory using FileLink. "
406 406 "Use FileLinks to display '%s'." % path)
407 407 self.path = fsdecode(path)
408 408 self.url_prefix = url_prefix
409 409 self.result_html_prefix = result_html_prefix
410 410 self.result_html_suffix = result_html_suffix
411 411
412 412 def _format_path(self):
413 413 fp = ''.join([self.url_prefix, html_escape(self.path)])
414 414 return ''.join([self.result_html_prefix,
415 415 self.html_link_str % \
416 416 (fp, html_escape(self.path, quote=False)),
417 417 self.result_html_suffix])
418 418
419 419 def _repr_html_(self):
420 420 """return html link to file
421 421 """
422 422 if not exists(self.path):
423 423 return ("Path (<tt>%s</tt>) doesn't exist. "
424 424 "It may still be in the process of "
425 425 "being generated, or you may have the "
426 426 "incorrect path." % self.path)
427 427
428 428 return self._format_path()
429 429
430 430 def __repr__(self):
431 431 """return absolute path to file
432 432 """
433 433 return abspath(self.path)
434 434
435 435 class FileLinks(FileLink):
436 436 """Class for embedding local file links in an IPython session, based on path
437 437
438 438 e.g. to embed links to files that were generated in the IPython notebook
439 439 under ``my/data``, you would do::
440 440
441 441 local_files = FileLinks("my/data")
442 442 display(local_files)
443 443
444 444 or in the HTML notebook, just::
445 445
446 446 FileLinks("my/data")
447 447 """
448 448 def __init__(self,
449 449 path,
450 450 url_prefix='',
451 451 included_suffixes=None,
452 452 result_html_prefix='',
453 453 result_html_suffix='<br>',
454 454 notebook_display_formatter=None,
455 455 terminal_display_formatter=None,
456 456 recursive=True):
457 457 """
458 458 See :class:`FileLink` for the ``path``, ``url_prefix``,
459 459 ``result_html_prefix`` and ``result_html_suffix`` parameters.
460 460
461 461 included_suffixes : list
462 462 Filename suffixes to include when formatting output [default: include
463 463 all files]
464 464
465 465 notebook_display_formatter : function
466 466 Used to format links for display in the notebook. See discussion of
467 467 formatter functions below.
468 468
469 469 terminal_display_formatter : function
470 470 Used to format links for display in the terminal. See discussion of
471 471 formatter functions below.
472 472
473 473 Formatter functions must be of the form::
474 474
475 475 f(dirname, fnames, included_suffixes)
476 476
477 477 dirname : str
478 478 The name of a directory
479 479 fnames : list
480 480 The files in that directory
481 481 included_suffixes : list
482 482 The file suffixes that should be included in the output (passing None
483 483 meansto include all suffixes in the output in the built-in formatters)
484 484 recursive : boolean
485 485 Whether to recurse into subdirectories. Default is True.
486 486
487 487 The function should return a list of lines that will be printed in the
488 488 notebook (if passing notebook_display_formatter) or the terminal (if
489 489 passing terminal_display_formatter). This function is iterated over for
490 490 each directory in self.path. Default formatters are in place, can be
491 491 passed here to support alternative formatting.
492 492
493 493 """
494 494 if isfile(path):
495 495 raise ValueError("Cannot display a file using FileLinks. "
496 496 "Use FileLink to display '%s'." % path)
497 497 self.included_suffixes = included_suffixes
498 498 # remove trailing slashes for more consistent output formatting
499 499 path = path.rstrip('/')
500 500
501 501 self.path = path
502 502 self.url_prefix = url_prefix
503 503 self.result_html_prefix = result_html_prefix
504 504 self.result_html_suffix = result_html_suffix
505 505
506 506 self.notebook_display_formatter = \
507 507 notebook_display_formatter or self._get_notebook_display_formatter()
508 508 self.terminal_display_formatter = \
509 509 terminal_display_formatter or self._get_terminal_display_formatter()
510 510
511 511 self.recursive = recursive
512 512
513 def _get_display_formatter(self,
514 dirname_output_format,
515 fname_output_format,
516 fp_format,
517 fp_cleaner=None):
518 """ generate built-in formatter function
513 def _get_display_formatter(
514 self, dirname_output_format, fname_output_format, fp_format, fp_cleaner=None
515 ):
516 """generate built-in formatter function
519 517
520 518 this is used to define both the notebook and terminal built-in
521 519 formatters as they only differ by some wrapper text for each entry
522 520
523 521 dirname_output_format: string to use for formatting directory
524 522 names, dirname will be substituted for a single "%s" which
525 523 must appear in this string
526 524 fname_output_format: string to use for formatting file names,
527 525 if a single "%s" appears in the string, fname will be substituted
528 526 if two "%s" appear in the string, the path to fname will be
529 527 substituted for the first and fname will be substituted for the
530 528 second
531 529 fp_format: string to use for formatting filepaths, must contain
532 530 exactly two "%s" and the dirname will be substituted for the first
533 531 and fname will be substituted for the second
534 532 """
535 533 def f(dirname, fnames, included_suffixes=None):
536 534 result = []
537 535 # begin by figuring out which filenames, if any,
538 536 # are going to be displayed
539 537 display_fnames = []
540 538 for fname in fnames:
541 539 if (isfile(join(dirname,fname)) and
542 540 (included_suffixes is None or
543 541 splitext(fname)[1] in included_suffixes)):
544 542 display_fnames.append(fname)
545 543
546 544 if len(display_fnames) == 0:
547 545 # if there are no filenames to display, don't print anything
548 546 # (not even the directory name)
549 547 pass
550 548 else:
551 549 # otherwise print the formatted directory name followed by
552 550 # the formatted filenames
553 551 dirname_output_line = dirname_output_format % dirname
554 552 result.append(dirname_output_line)
555 553 for fname in display_fnames:
556 554 fp = fp_format % (dirname,fname)
557 555 if fp_cleaner is not None:
558 556 fp = fp_cleaner(fp)
559 557 try:
560 558 # output can include both a filepath and a filename...
561 559 fname_output_line = fname_output_format % (fp, fname)
562 560 except TypeError:
563 561 # ... or just a single filepath
564 562 fname_output_line = fname_output_format % fname
565 563 result.append(fname_output_line)
566 564 return result
567 565 return f
568 566
569 567 def _get_notebook_display_formatter(self,
570 568 spacer="&nbsp;&nbsp;"):
571 569 """ generate function to use for notebook formatting
572 570 """
573 571 dirname_output_format = \
574 572 self.result_html_prefix + "%s/" + self.result_html_suffix
575 573 fname_output_format = \
576 574 self.result_html_prefix + spacer + self.html_link_str + self.result_html_suffix
577 575 fp_format = self.url_prefix + '%s/%s'
578 576 if sep == "\\":
579 577 # Working on a platform where the path separator is "\", so
580 578 # must convert these to "/" for generating a URI
581 579 def fp_cleaner(fp):
582 580 # Replace all occurrences of backslash ("\") with a forward
583 581 # slash ("/") - this is necessary on windows when a path is
584 582 # provided as input, but we must link to a URI
585 583 return fp.replace('\\','/')
586 584 else:
587 585 fp_cleaner = None
588 586
589 587 return self._get_display_formatter(dirname_output_format,
590 588 fname_output_format,
591 589 fp_format,
592 590 fp_cleaner)
593 591
594 592 def _get_terminal_display_formatter(self,
595 593 spacer=" "):
596 594 """ generate function to use for terminal formatting
597 595 """
598 596 dirname_output_format = "%s/"
599 597 fname_output_format = spacer + "%s"
600 598 fp_format = '%s/%s'
601 599
602 600 return self._get_display_formatter(dirname_output_format,
603 601 fname_output_format,
604 602 fp_format)
605 603
606 604 def _format_path(self):
607 605 result_lines = []
608 606 if self.recursive:
609 607 walked_dir = list(walk(self.path))
610 608 else:
611 609 walked_dir = [next(walk(self.path))]
612 610 walked_dir.sort()
613 611 for dirname, subdirs, fnames in walked_dir:
614 612 result_lines += self.notebook_display_formatter(dirname, fnames, self.included_suffixes)
615 613 return '\n'.join(result_lines)
616 614
617 615 def __repr__(self):
618 616 """return newline-separated absolute paths
619 617 """
620 618 result_lines = []
621 619 if self.recursive:
622 620 walked_dir = list(walk(self.path))
623 621 else:
624 622 walked_dir = [next(walk(self.path))]
625 623 walked_dir.sort()
626 624 for dirname, subdirs, fnames in walked_dir:
627 625 result_lines += self.terminal_display_formatter(dirname, fnames, self.included_suffixes)
628 626 return '\n'.join(result_lines)
629 627
630 628
631 629 class Code(TextDisplayObject):
632 630 """Display syntax-highlighted source code.
633 631
634 632 This uses Pygments to highlight the code for HTML and Latex output.
635 633
636 634 Parameters
637 635 ----------
638 636 data : str
639 637 The code as a string
640 638 url : str
641 639 A URL to fetch the code from
642 640 filename : str
643 641 A local filename to load the code from
644 642 language : str
645 643 The short name of a Pygments lexer to use for highlighting.
646 644 If not specified, it will guess the lexer based on the filename
647 645 or the code. Available lexers: http://pygments.org/docs/lexers/
648 646 """
649 647 def __init__(self, data=None, url=None, filename=None, language=None):
650 648 self.language = language
651 649 super().__init__(data=data, url=url, filename=filename)
652 650
653 651 def _get_lexer(self):
654 652 if self.language:
655 653 from pygments.lexers import get_lexer_by_name
656 654 return get_lexer_by_name(self.language)
657 655 elif self.filename:
658 656 from pygments.lexers import get_lexer_for_filename
659 657 return get_lexer_for_filename(self.filename)
660 658 else:
661 659 from pygments.lexers import guess_lexer
662 660 return guess_lexer(self.data)
663 661
664 662 def __repr__(self):
665 663 return self.data
666 664
667 665 def _repr_html_(self):
668 666 from pygments import highlight
669 667 from pygments.formatters import HtmlFormatter
670 668 fmt = HtmlFormatter()
671 669 style = '<style>{}</style>'.format(fmt.get_style_defs('.output_html'))
672 670 return style + highlight(self.data, self._get_lexer(), fmt)
673 671
674 672 def _repr_latex_(self):
675 673 from pygments import highlight
676 674 from pygments.formatters import LatexFormatter
677 675 return highlight(self.data, self._get_lexer(), LatexFormatter())
@@ -1,192 +1,192 b''
1 1 """Tests for IPython.utils.path.py"""
2 2 # Copyright (c) IPython Development Team.
3 3 # Distributed under the terms of the Modified BSD License.
4 4
5 5 from contextlib import contextmanager
6 6 from unittest.mock import patch
7 7
8 8 import pytest
9 9
10 10 from IPython.lib import latextools
11 11 from IPython.testing.decorators import (
12 12 onlyif_cmds_exist,
13 13 skipif_not_matplotlib,
14 14 )
15 15 from IPython.utils.process import FindCmdError
16 16
17 17
18 18 @pytest.mark.parametrize('command', ['latex', 'dvipng'])
19 19 def test_check_latex_to_png_dvipng_fails_when_no_cmd(command):
20 20 def mock_find_cmd(arg):
21 21 if arg == command:
22 22 raise FindCmdError
23 23
24 24 with patch.object(latextools, "find_cmd", mock_find_cmd):
25 25 assert latextools.latex_to_png_dvipng("whatever", True) is None
26 26
27 27
28 28 @contextmanager
29 29 def no_op(*args, **kwargs):
30 30 yield
31 31
32 32
33 33 @onlyif_cmds_exist("latex", "dvipng")
34 @pytest.mark.parametrize("s, wrap", [(u"$$x^2$$", False), (u"x^2", True)])
34 @pytest.mark.parametrize("s, wrap", [("$$x^2$$", False), ("x^2", True)])
35 35 def test_latex_to_png_dvipng_runs(s, wrap):
36 36 """
37 37 Test that latex_to_png_dvipng just runs without error.
38 38 """
39 39 def mock_kpsewhich(filename):
40 40 assert filename == "breqn.sty"
41 41 return None
42 42
43 43 latextools.latex_to_png_dvipng(s, wrap)
44 44
45 45 with patch_latextool(mock_kpsewhich):
46 46 latextools.latex_to_png_dvipng(s, wrap)
47 47
48 48
49 49 def mock_kpsewhich(filename):
50 50 assert filename == "breqn.sty"
51 51 return None
52 52
53 53 @contextmanager
54 54 def patch_latextool(mock=mock_kpsewhich):
55 55 with patch.object(latextools, "kpsewhich", mock):
56 56 yield
57 57
58 58 @pytest.mark.parametrize('context', [no_op, patch_latextool])
59 59 @pytest.mark.parametrize('s_wrap', [("$x^2$", False), ("x^2", True)])
60 60 def test_latex_to_png_mpl_runs(s_wrap, context):
61 61 """
62 62 Test that latex_to_png_mpl just runs without error.
63 63 """
64 64 try:
65 65 import matplotlib
66 66 except ImportError:
67 67 pytest.skip("This needs matplotlib to be available")
68 68 return
69 69 s, wrap = s_wrap
70 70 with context():
71 71 latextools.latex_to_png_mpl(s, wrap)
72 72
73 73 @skipif_not_matplotlib
74 74 def test_latex_to_html():
75 75 img = latextools.latex_to_html("$x^2$")
76 76 assert "data:image/png;base64,iVBOR" in img
77 77
78 78
79 79 def test_genelatex_no_wrap():
80 80 """
81 81 Test genelatex with wrap=False.
82 82 """
83 83 def mock_kpsewhich(filename):
84 84 assert False, ("kpsewhich should not be called "
85 85 "(called with {0})".format(filename))
86 86
87 87 with patch_latextool(mock_kpsewhich):
88 88 assert '\n'.join(latextools.genelatex("body text", False)) == r'''\documentclass{article}
89 89 \usepackage{amsmath}
90 90 \usepackage{amsthm}
91 91 \usepackage{amssymb}
92 92 \usepackage{bm}
93 93 \pagestyle{empty}
94 94 \begin{document}
95 95 body text
96 96 \end{document}'''
97 97
98 98
99 99 def test_genelatex_wrap_with_breqn():
100 100 """
101 101 Test genelatex with wrap=True for the case breqn.sty is installed.
102 102 """
103 103 def mock_kpsewhich(filename):
104 104 assert filename == "breqn.sty"
105 105 return "path/to/breqn.sty"
106 106
107 107 with patch_latextool(mock_kpsewhich):
108 108 assert '\n'.join(latextools.genelatex("x^2", True)) == r'''\documentclass{article}
109 109 \usepackage{amsmath}
110 110 \usepackage{amsthm}
111 111 \usepackage{amssymb}
112 112 \usepackage{bm}
113 113 \usepackage{breqn}
114 114 \pagestyle{empty}
115 115 \begin{document}
116 116 \begin{dmath*}
117 117 x^2
118 118 \end{dmath*}
119 119 \end{document}'''
120 120
121 121
122 122 def test_genelatex_wrap_without_breqn():
123 123 """
124 124 Test genelatex with wrap=True for the case breqn.sty is not installed.
125 125 """
126 126 def mock_kpsewhich(filename):
127 127 assert filename == "breqn.sty"
128 128 return None
129 129
130 130 with patch_latextool(mock_kpsewhich):
131 131 assert '\n'.join(latextools.genelatex("x^2", True)) == r'''\documentclass{article}
132 132 \usepackage{amsmath}
133 133 \usepackage{amsthm}
134 134 \usepackage{amssymb}
135 135 \usepackage{bm}
136 136 \pagestyle{empty}
137 137 \begin{document}
138 138 $$x^2$$
139 139 \end{document}'''
140 140
141 141
142 142 @skipif_not_matplotlib
143 143 @onlyif_cmds_exist('latex', 'dvipng')
144 144 def test_latex_to_png_color():
145 145 """
146 146 Test color settings for latex_to_png.
147 147 """
148 148 latex_string = "$x^2$"
149 149 default_value = latextools.latex_to_png(latex_string, wrap=False)
150 150 default_hexblack = latextools.latex_to_png(latex_string, wrap=False,
151 151 color='#000000')
152 152 dvipng_default = latextools.latex_to_png_dvipng(latex_string, False)
153 153 dvipng_black = latextools.latex_to_png_dvipng(latex_string, False, 'Black')
154 154 assert dvipng_default == dvipng_black
155 155 mpl_default = latextools.latex_to_png_mpl(latex_string, False)
156 156 mpl_black = latextools.latex_to_png_mpl(latex_string, False, 'Black')
157 157 assert mpl_default == mpl_black
158 158 assert default_value in [dvipng_black, mpl_black]
159 159 assert default_hexblack in [dvipng_black, mpl_black]
160 160
161 161 # Test that dvips name colors can be used without error
162 162 dvipng_maroon = latextools.latex_to_png_dvipng(latex_string, False,
163 163 'Maroon')
164 164 # And that it doesn't return the black one
165 165 assert dvipng_black != dvipng_maroon
166 166
167 167 mpl_maroon = latextools.latex_to_png_mpl(latex_string, False, 'Maroon')
168 168 assert mpl_black != mpl_maroon
169 169 mpl_white = latextools.latex_to_png_mpl(latex_string, False, 'White')
170 170 mpl_hexwhite = latextools.latex_to_png_mpl(latex_string, False, '#FFFFFF')
171 171 assert mpl_white == mpl_hexwhite
172 172
173 173 mpl_white_scale = latextools.latex_to_png_mpl(latex_string, False,
174 174 'White', 1.2)
175 175 assert mpl_white != mpl_white_scale
176 176
177 177
178 178 def test_latex_to_png_invalid_hex_colors():
179 179 """
180 180 Test that invalid hex colors provided to dvipng gives an exception.
181 181 """
182 182 latex_string = "$x^2$"
183 183 pytest.raises(
184 184 ValueError,
185 185 lambda: latextools.latex_to_png(
186 186 latex_string, backend="dvipng", color="#f00bar"
187 187 ),
188 188 )
189 189 pytest.raises(
190 190 ValueError,
191 191 lambda: latextools.latex_to_png(latex_string, backend="dvipng", color="#f00"),
192 192 )
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
General Comments 0
You need to be logged in to leave comments. Login now