Remove unneeded `skip_doctest` decorator usages

IPython/core/debugger.py

@@ -1,1109 +1,1106 b''
 # -*- coding: utf-8 -*-
 """
 Pdb debugger class.
 
 
 This is an extension to PDB which adds a number of new features.
 Note that there is also the `IPython.terminal.debugger` class which provides UI
 improvements.
 
 We also strongly recommend to use this via the `ipdb` package, which provides
 extra configuration options.
 
 Among other things, this subclass of PDB:
  - supports many IPython magics like pdef/psource
  - hide frames in tracebacks based on `__tracebackhide__`
  - allows to skip frames based on `__debuggerskip__`
 
 The skipping and hiding frames are configurable via the `skip_predicates`
 command.
 
 By default, frames from readonly files will be hidden, frames containing
 ``__tracebackhide__=True`` will be hidden.
 
 Frames containing ``__debuggerskip__`` will be stepped over, frames who's parent
 frames value of ``__debuggerskip__`` is ``True`` will be skipped.
 
     >>> def helpers_helper():
     ...     pass
     ...
     ... def helper_1():
     ...     print("don't step in me")
     ...     helpers_helpers() # will be stepped over unless breakpoint set.
     ...
     ...
     ... def helper_2():
     ...     print("in me neither")
     ...
 
 One can define a decorator that wraps a function between the two helpers:
 
     >>> def pdb_skipped_decorator(function):
     ...
     ...
     ...     def wrapped_fn(*args, **kwargs):
     ...         __debuggerskip__ = True
     ...         helper_1()
     ...         __debuggerskip__ = False
     ...         result = function(*args, **kwargs)
     ...         __debuggerskip__ = True
     ...         helper_2()
     ...         # setting __debuggerskip__ to False again is not necessary
     ...         return result
     ...
     ...     return wrapped_fn
 
 When decorating a function, ipdb will directly step into ``bar()`` by
 default:
 
     >>> @foo_decorator
     ... def bar(x, y):
     ...     return x * y
 
 
 You can toggle the behavior with
 
     ipdb> skip_predicates debuggerskip false
 
 or configure it in your ``.pdbrc``
 
 
 
 License
 -------
 
 Modified from the standard pdb.Pdb class to avoid including readline, so that
 the command line completion of other programs which include this isn't
 damaged.
 
 In the future, this class will be expanded with improvements over the standard
 pdb.
 
 The original code in this file is mainly lifted out of cmd.py in Python 2.2,
 with minor changes. Licensing should therefore be under the standard Python
 terms.  For details on the PSF (Python Software Foundation) standard license,
 see:
 
 https://docs.python.org/2/license.html
 
 
 All the changes since then are under the same license as IPython.
 
 """
 
 #*****************************************************************************
 #
 #       This file is licensed under the PSF license.
 #
 #       Copyright (C) 2001 Python Software Foundation, www.python.org
 #       Copyright (C) 2005-2006 Fernando Perez. <[e-mail]>
 #
 #
 #*****************************************************************************
 
 import bdb
 import functools
 import inspect
 import linecache
 import sys
 import warnings
 import re
 import os
 
 from IPython import get_ipython
 from IPython.utils import PyColorize
 from IPython.utils import coloransi, py3compat
 from IPython.core.excolors import exception_colors
-from IPython.testing.skipdoctest import skip_doctest
 
 # skip module docstests
 __skip_doctest__ = True
 
 prompt = 'ipdb> '
 
 # We have to check this directly from sys.argv, config struct not yet available
 from pdb import Pdb as OldPdb
 
 # Allow the set_trace code to operate outside of an ipython instance, even if
 # it does so with some limitations.  The rest of this support is implemented in
 # the Tracer constructor.
 
 DEBUGGERSKIP = "__debuggerskip__"
 
 
 def make_arrow(pad):
     """generate the leading arrow in front of traceback or debugger"""
     if pad >= 2:
         return '-'*(pad-2) + '> '
     elif pad == 1:
         return '>'
     return ''
 
 
 def BdbQuit_excepthook(et, ev, tb, excepthook=None):
     """Exception hook which handles `BdbQuit` exceptions.
 
     All other exceptions are processed using the `excepthook`
     parameter.
     """
     warnings.warn("`BdbQuit_excepthook` is deprecated since version 5.1",
                   DeprecationWarning, stacklevel=2)
     if et == bdb.BdbQuit:
         print('Exiting Debugger.')
     elif excepthook is not None:
         excepthook(et, ev, tb)
     else:
         # Backwards compatibility. Raise deprecation warning?
         BdbQuit_excepthook.excepthook_ori(et, ev, tb)
 
 
 def BdbQuit_IPython_excepthook(self, et, ev, tb, tb_offset=None):
     warnings.warn(
         "`BdbQuit_IPython_excepthook` is deprecated since version 5.1",
         DeprecationWarning, stacklevel=2)
     print('Exiting Debugger.')
 
 
 class Tracer(object):
     """
     DEPRECATED
 
     Class for local debugging, similar to pdb.set_trace.
 
     Instances of this class, when called, behave like pdb.set_trace, but
     providing IPython's enhanced capabilities.
 
     This is implemented as a class which must be initialized in your own code
     and not as a standalone function because we need to detect at runtime
     whether IPython is already active or not.  That detection is done in the
     constructor, ensuring that this code plays nicely with a running IPython,
     while functioning acceptably (though with limitations) if outside of it.
     """
 
-    @skip_doctest
     def __init__(self, colors=None):
         """
         DEPRECATED
 
         Create a local debugger instance.
 
         Parameters
         ----------
         colors : str, optional
             The name of the color scheme to use, it must be one of IPython's
             valid color schemes.  If not given, the function will default to
             the current IPython scheme when running inside IPython, and to
             'NoColor' otherwise.
 
         Examples
         --------
         ::
 
             from IPython.core.debugger import Tracer; debug_here = Tracer()
 
         Later in your code::
 
             debug_here()  # -> will open up the debugger at that point.
 
         Once the debugger activates, you can use all of its regular commands to
         step through code, set breakpoints, etc.  See the pdb documentation
         from the Python standard library for usage details.
         """
         warnings.warn("`Tracer` is deprecated since version 5.1, directly use "
                       "`IPython.core.debugger.Pdb.set_trace()`",
                       DeprecationWarning, stacklevel=2)
 
         ip = get_ipython()
         if ip is None:
             # Outside of ipython, we set our own exception hook manually
             sys.excepthook = functools.partial(BdbQuit_excepthook,
                                                excepthook=sys.excepthook)
             def_colors = 'NoColor'
         else:
             # In ipython, we use its custom exception handler mechanism
             def_colors = ip.colors
             ip.set_custom_exc((bdb.BdbQuit,), BdbQuit_IPython_excepthook)
 
         if colors is None:
             colors = def_colors
 
         # The stdlib debugger internally uses a modified repr from the `repr`
         # module, that limits the length of printed strings to a hardcoded
         # limit of 30 characters.  That much trimming is too aggressive, let's
         # at least raise that limit to 80 chars, which should be enough for
         # most interactive uses.
         try:
             from reprlib import aRepr
             aRepr.maxstring = 80
         except:
             # This is only a user-facing convenience, so any error we encounter
             # here can be warned about but can be otherwise ignored.  These
             # printouts will tell us about problems if this API changes
             import traceback
             traceback.print_exc()
 
         self.debugger = Pdb(colors)
 
     def __call__(self):
         """Starts an interactive debugger at the point where called.
 
         This is similar to the pdb.set_trace() function from the std lib, but
         using IPython's enhanced debugger."""
 
         self.debugger.set_trace(sys._getframe().f_back)
 
 
 RGX_EXTRA_INDENT = re.compile(r'(?<=\n)\s+')
 
 
 def strip_indentation(multiline_string):
     return RGX_EXTRA_INDENT.sub('', multiline_string)
 
 
 def decorate_fn_with_doc(new_fn, old_fn, additional_text=""):
     """Make new_fn have old_fn's doc string. This is particularly useful
     for the ``do_...`` commands that hook into the help system.
     Adapted from from a comp.lang.python posting
     by Duncan Booth."""
     def wrapper(*args, **kw):
         return new_fn(*args, **kw)
     if old_fn.__doc__:
         wrapper.__doc__ = strip_indentation(old_fn.__doc__) + additional_text
     return wrapper
 
 
 class Pdb(OldPdb):
     """Modified Pdb class, does not load readline.
 
     for a standalone version that uses prompt_toolkit, see
     `IPython.terminal.debugger.TerminalPdb` and
     `IPython.terminal.debugger.set_trace()`
 
 
     This debugger can hide and skip frames that are tagged according to some predicates.
     See the `skip_predicates` commands.
 
     """
 
     default_predicates = {
         "tbhide": True,
         "readonly": False,
         "ipython_internal": True,
         "debuggerskip": True,
     }
 
     def __init__(self, color_scheme=None, completekey=None,
                  stdin=None, stdout=None, context=5, **kwargs):
         """Create a new IPython debugger.
 
         Parameters
         ----------
         color_scheme : default None
             Deprecated, do not use.
         completekey : default None
             Passed to pdb.Pdb.
         stdin : default None
             Passed to pdb.Pdb.
         stdout : default None
             Passed to pdb.Pdb.
         context : int
             Number of lines of source code context to show when
             displaying stacktrace information.
         **kwargs
             Passed to pdb.Pdb.
 
         Notes
         -----
         The possibilities are python version dependent, see the python
         docs for more info.
         """
 
         # Parent constructor:
         try:
             self.context = int(context)
             if self.context <= 0:
                 raise ValueError("Context must be a positive integer")
         except (TypeError, ValueError) as e:
                 raise ValueError("Context must be a positive integer") from e
 
         # `kwargs` ensures full compatibility with stdlib's `pdb.Pdb`.
         OldPdb.__init__(self, completekey, stdin, stdout, **kwargs)
 
         # IPython changes...
         self.shell = get_ipython()
 
         if self.shell is None:
             save_main = sys.modules['__main__']
             # No IPython instance running, we must create one
             from IPython.terminal.interactiveshell import \
                 TerminalInteractiveShell
             self.shell = TerminalInteractiveShell.instance()
             # needed by any code which calls __import__("__main__") after
             # the debugger was entered. See also #9941.
             sys.modules["__main__"] = save_main
 
         if color_scheme is not None:
             warnings.warn(
                 "The `color_scheme` argument is deprecated since version 5.1",
                 DeprecationWarning, stacklevel=2)
         else:
             color_scheme = self.shell.colors
 
         self.aliases = {}
 
         # Create color table: we copy the default one from the traceback
         # module and add a few attributes needed for debugging
         self.color_scheme_table = exception_colors()
 
         # shorthands
         C = coloransi.TermColors
         cst = self.color_scheme_table
 
         cst['NoColor'].colors.prompt = C.NoColor
         cst['NoColor'].colors.breakpoint_enabled = C.NoColor
         cst['NoColor'].colors.breakpoint_disabled = C.NoColor
 
         cst['Linux'].colors.prompt = C.Green
         cst['Linux'].colors.breakpoint_enabled = C.LightRed
         cst['Linux'].colors.breakpoint_disabled = C.Red
 
         cst['LightBG'].colors.prompt = C.Blue
         cst['LightBG'].colors.breakpoint_enabled = C.LightRed
         cst['LightBG'].colors.breakpoint_disabled = C.Red
 
         cst['Neutral'].colors.prompt = C.Blue
         cst['Neutral'].colors.breakpoint_enabled = C.LightRed
         cst['Neutral'].colors.breakpoint_disabled = C.Red
 
         # Add a python parser so we can syntax highlight source while
         # debugging.
         self.parser = PyColorize.Parser(style=color_scheme)
         self.set_colors(color_scheme)
 
         # Set the prompt - the default prompt is '(Pdb)'
         self.prompt = prompt
         self.skip_hidden = True
         self.report_skipped = True
 
         # list of predicates we use to skip frames
         self._predicates = self.default_predicates
 
     #
     def set_colors(self, scheme):
         """Shorthand access to the color table scheme selector method."""
         self.color_scheme_table.set_active_scheme(scheme)
         self.parser.style = scheme
 
     def set_trace(self, frame=None):
         if frame is None:
             frame = sys._getframe().f_back
         self.initial_frame = frame
         return super().set_trace(frame)
 
     def _hidden_predicate(self, frame):
         """
         Given a frame return whether it it should be hidden or not by IPython.
         """
 
         if self._predicates["readonly"]:
             fname = frame.f_code.co_filename
             # we need to check for file existence and interactively define
             # function would otherwise appear as RO.
             if os.path.isfile(fname) and not os.access(fname, os.W_OK):
                 return True
 
         if self._predicates["tbhide"]:
             if frame in (self.curframe, getattr(self, "initial_frame", None)):
                 return False
             frame_locals = self._get_frame_locals(frame)
             if "__tracebackhide__" not in frame_locals:
                 return False
             return frame_locals["__tracebackhide__"]
         return False
 
     def hidden_frames(self, stack):
         """
         Given an index in the stack return whether it should be skipped.
 
         This is used in up/down and where to skip frames.
         """
         # The f_locals dictionary is updated from the actual frame
         # locals whenever the .f_locals accessor is called, so we
         # avoid calling it here to preserve self.curframe_locals.
         # Furthermore, there is no good reason to hide the current frame.
         ip_hide = [self._hidden_predicate(s[0]) for s in stack]
         ip_start = [i for i, s in enumerate(ip_hide) if s == "__ipython_bottom__"]
         if ip_start and self._predicates["ipython_internal"]:
             ip_hide = [h if i > ip_start[0] else True for (i, h) in enumerate(ip_hide)]
         return ip_hide
 
     def interaction(self, frame, traceback):
         try:
             OldPdb.interaction(self, frame, traceback)
         except KeyboardInterrupt:
             self.stdout.write("\n" + self.shell.get_exception_only())
 
     def precmd(self, line):
         """Perform useful escapes on the command before it is executed."""
 
         if line.endswith("??"):
             line = "pinfo2 " + line[:-2]
         elif line.endswith("?"):
             line = "pinfo " + line[:-1]
 
         line = super().precmd(line)
 
         return line
 
     def new_do_frame(self, arg):
         OldPdb.do_frame(self, arg)
 
     def new_do_quit(self, arg):
 
         if hasattr(self, 'old_all_completions'):
             self.shell.Completer.all_completions = self.old_all_completions
 
         return OldPdb.do_quit(self, arg)
 
     do_q = do_quit = decorate_fn_with_doc(new_do_quit, OldPdb.do_quit)
 
     def new_do_restart(self, arg):
         """Restart command. In the context of ipython this is exactly the same
         thing as 'quit'."""
         self.msg("Restart doesn't make sense here. Using 'quit' instead.")
         return self.do_quit(arg)
 
     def print_stack_trace(self, context=None):
         Colors = self.color_scheme_table.active_colors
         ColorsNormal = Colors.Normal
         if context is None:
             context = self.context
         try:
             context = int(context)
             if context <= 0:
                 raise ValueError("Context must be a positive integer")
         except (TypeError, ValueError) as e:
                 raise ValueError("Context must be a positive integer") from e
         try:
             skipped = 0
             for hidden, frame_lineno in zip(self.hidden_frames(self.stack), self.stack):
                 if hidden and self.skip_hidden:
                     skipped += 1
                     continue
                 if skipped:
                     print(
                         f"{Colors.excName}    [... skipping {skipped} hidden frame(s)]{ColorsNormal}\n"
                     )
                     skipped = 0
                 self.print_stack_entry(frame_lineno, context=context)
             if skipped:
                 print(
                     f"{Colors.excName}    [... skipping {skipped} hidden frame(s)]{ColorsNormal}\n"
                 )
         except KeyboardInterrupt:
             pass
 
     def print_stack_entry(self, frame_lineno, prompt_prefix='\n-> ',
                           context=None):
         if context is None:
             context = self.context
         try:
             context = int(context)
             if context <= 0:
                 raise ValueError("Context must be a positive integer")
         except (TypeError, ValueError) as e:
                 raise ValueError("Context must be a positive integer") from e
         print(self.format_stack_entry(frame_lineno, '', context), file=self.stdout)
 
         # vds: >>
         frame, lineno = frame_lineno
         filename = frame.f_code.co_filename
         self.shell.hooks.synchronize_with_editor(filename, lineno, 0)
         # vds: <<
 
     def _get_frame_locals(self, frame):
         """ "
         Accessing f_local of current frame reset the namespace, so we want to avoid
         that or the following can happen
 
         ipdb> foo
         "old"
         ipdb> foo = "new"
         ipdb> foo
         "new"
         ipdb> where
         ipdb> foo
         "old"
 
         So if frame is self.current_frame we instead return self.curframe_locals
 
         """
         if frame is self.curframe:
             return self.curframe_locals
         else:
             return frame.f_locals
 
     def format_stack_entry(self, frame_lineno, lprefix=': ', context=None):
         if context is None:
             context = self.context
         try:
             context = int(context)
             if context <= 0:
                 print("Context must be a positive integer", file=self.stdout)
         except (TypeError, ValueError):
                 print("Context must be a positive integer", file=self.stdout)
 
         import reprlib
 
         ret = []
 
         Colors = self.color_scheme_table.active_colors
         ColorsNormal = Colors.Normal
         tpl_link = "%s%%s%s" % (Colors.filenameEm, ColorsNormal)
         tpl_call = "%s%%s%s%%s%s" % (Colors.vName, Colors.valEm, ColorsNormal)
         tpl_line = "%%s%s%%s %s%%s" % (Colors.lineno, ColorsNormal)
         tpl_line_em = "%%s%s%%s %s%%s%s" % (Colors.linenoEm, Colors.line, ColorsNormal)
 
         frame, lineno = frame_lineno
 
         return_value = ''
         loc_frame = self._get_frame_locals(frame)
         if "__return__" in loc_frame:
             rv = loc_frame["__return__"]
             # return_value += '->'
             return_value += reprlib.repr(rv) + "\n"
         ret.append(return_value)
 
         #s = filename + '(' + `lineno` + ')'
         filename = self.canonic(frame.f_code.co_filename)
         link = tpl_link % py3compat.cast_unicode(filename)
 
         if frame.f_code.co_name:
             func = frame.f_code.co_name
         else:
             func = "<lambda>"
 
         call = ""
         if func != "?":
             if "__args__" in loc_frame:
                 args = reprlib.repr(loc_frame["__args__"])
             else:
                 args = '()'
             call = tpl_call % (func, args)
 
         # The level info should be generated in the same format pdb uses, to
         # avoid breaking the pdbtrack functionality of python-mode in *emacs.
         if frame is self.curframe:
             ret.append('> ')
         else:
             ret.append("  ")
         ret.append("%s(%s)%s\n" % (link, lineno, call))
 
         start = lineno - 1 - context//2
         lines = linecache.getlines(filename)
         start = min(start, len(lines) - context)
         start = max(start, 0)
         lines = lines[start : start + context]
 
         for i, line in enumerate(lines):
             show_arrow = start + 1 + i == lineno
             linetpl = (frame is self.curframe or show_arrow) and tpl_line_em or tpl_line
             ret.append(
                 self.__format_line(
                     linetpl, filename, start + 1 + i, line, arrow=show_arrow
                 )
             )
         return "".join(ret)
 
     def __format_line(self, tpl_line, filename, lineno, line, arrow=False):
         bp_mark = ""
         bp_mark_color = ""
 
         new_line, err = self.parser.format2(line, 'str')
         if not err:
             line = new_line
 
         bp = None
         if lineno in self.get_file_breaks(filename):
             bps = self.get_breaks(filename, lineno)
             bp = bps[-1]
 
         if bp:
             Colors = self.color_scheme_table.active_colors
             bp_mark = str(bp.number)
             bp_mark_color = Colors.breakpoint_enabled
             if not bp.enabled:
                 bp_mark_color = Colors.breakpoint_disabled
 
         numbers_width = 7
         if arrow:
             # This is the line with the error
             pad = numbers_width - len(str(lineno)) - len(bp_mark)
             num = '%s%s' % (make_arrow(pad), str(lineno))
         else:
             num = '%*s' % (numbers_width - len(bp_mark), str(lineno))
 
         return tpl_line % (bp_mark_color + bp_mark, num, line)
 
     def print_list_lines(self, filename, first, last):
         """The printing (as opposed to the parsing part of a 'list'
         command."""
         try:
             Colors = self.color_scheme_table.active_colors
             ColorsNormal = Colors.Normal
             tpl_line = '%%s%s%%s %s%%s' % (Colors.lineno, ColorsNormal)
             tpl_line_em = '%%s%s%%s %s%%s%s' % (Colors.linenoEm, Colors.line, ColorsNormal)
             src = []
             if filename == "<string>" and hasattr(self, "_exec_filename"):
                 filename = self._exec_filename
 
             for lineno in range(first, last+1):
                 line = linecache.getline(filename, lineno)
                 if not line:
                     break
 
                 if lineno == self.curframe.f_lineno:
                     line = self.__format_line(
                         tpl_line_em, filename, lineno, line, arrow=True
                     )
                 else:
                     line = self.__format_line(
                         tpl_line, filename, lineno, line, arrow=False
                     )
 
                 src.append(line)
                 self.lineno = lineno
 
             print(''.join(src), file=self.stdout)
 
         except KeyboardInterrupt:
             pass
 
     def do_skip_predicates(self, args):
         """
         Turn on/off individual predicates as to whether a frame should be hidden/skip.
 
         The global option to skip (or not) hidden frames is set with skip_hidden
 
         To change the value of a predicate
 
             skip_predicates key [true|false]
 
         Call without arguments to see the current values.
 
         To permanently change the value of an option add the corresponding
         command to your ``~/.pdbrc`` file. If you are programmatically using the
         Pdb instance you can also change the ``default_predicates`` class
         attribute.
         """
         if not args.strip():
             print("current predicates:")
             for (p, v) in self._predicates.items():
                 print("   ", p, ":", v)
             return
         type_value = args.strip().split(" ")
         if len(type_value) != 2:
             print(
                 f"Usage: skip_predicates <type> <value>, with <type> one of {set(self._predicates.keys())}"
             )
             return
 
         type_, value = type_value
         if type_ not in self._predicates:
             print(f"{type_!r} not in {set(self._predicates.keys())}")
             return
         if value.lower() not in ("true", "yes", "1", "no", "false", "0"):
             print(
                 f"{value!r} is invalid - use one of ('true', 'yes', '1', 'no', 'false', '0')"
             )
             return
 
         self._predicates[type_] = value.lower() in ("true", "yes", "1")
         if not any(self._predicates.values()):
             print(
                 "Warning, all predicates set to False, skip_hidden may not have any effects."
             )
 
     def do_skip_hidden(self, arg):
         """
         Change whether or not we should skip frames with the
         __tracebackhide__ attribute.
         """
         if not arg.strip():
             print(
                 f"skip_hidden = {self.skip_hidden}, use 'yes','no', 'true', or 'false' to change."
             )
         elif arg.strip().lower() in ("true", "yes"):
             self.skip_hidden = True
         elif arg.strip().lower() in ("false", "no"):
             self.skip_hidden = False
         if not any(self._predicates.values()):
             print(
                 "Warning, all predicates set to False, skip_hidden may not have any effects."
             )
 
     def do_list(self, arg):
         """Print lines of code from the current stack frame
         """
         self.lastcmd = 'list'
         last = None
         if arg:
             try:
                 x = eval(arg, {}, {})
                 if type(x) == type(()):
                     first, last = x
                     first = int(first)
                     last = int(last)
                     if last < first:
                         # Assume it's a count
                         last = first + last
                 else:
                     first = max(1, int(x) - 5)
             except:
                 print('*** Error in argument:', repr(arg), file=self.stdout)
                 return
         elif self.lineno is None:
             first = max(1, self.curframe.f_lineno - 5)
         else:
             first = self.lineno + 1
         if last is None:
             last = first + 10
         self.print_list_lines(self.curframe.f_code.co_filename, first, last)
 
         # vds: >>
         lineno = first
         filename = self.curframe.f_code.co_filename
         self.shell.hooks.synchronize_with_editor(filename, lineno, 0)
         # vds: <<
 
     do_l = do_list
 
     def getsourcelines(self, obj):
         lines, lineno = inspect.findsource(obj)
         if inspect.isframe(obj) and obj.f_globals is self._get_frame_locals(obj):
             # must be a module frame: do not try to cut a block out of it
             return lines, 1
         elif inspect.ismodule(obj):
             return lines, 1
         return inspect.getblock(lines[lineno:]), lineno+1
 
     def do_longlist(self, arg):
         """Print lines of code from the current stack frame.
 
         Shows more lines than 'list' does.
         """
         self.lastcmd = 'longlist'
         try:
             lines, lineno = self.getsourcelines(self.curframe)
         except OSError as err:
             self.error(err)
             return
         last = lineno + len(lines)
         self.print_list_lines(self.curframe.f_code.co_filename, lineno, last)
     do_ll = do_longlist
 
     def do_debug(self, arg):
         """debug code
         Enter a recursive debugger that steps through the code
         argument (which is an arbitrary expression or statement to be
         executed in the current environment).
         """
         trace_function = sys.gettrace()
         sys.settrace(None)
         globals = self.curframe.f_globals
         locals = self.curframe_locals
         p = self.__class__(completekey=self.completekey,
                            stdin=self.stdin, stdout=self.stdout)
         p.use_rawinput = self.use_rawinput
         p.prompt = "(%s) " % self.prompt.strip()
         self.message("ENTERING RECURSIVE DEBUGGER")
         sys.call_tracing(p.run, (arg, globals, locals))
         self.message("LEAVING RECURSIVE DEBUGGER")
         sys.settrace(trace_function)
         self.lastcmd = p.lastcmd
 
     def do_pdef(self, arg):
         """Print the call signature for any callable object.
 
         The debugger interface to %pdef"""
         namespaces = [
             ("Locals", self.curframe_locals),
             ("Globals", self.curframe.f_globals),
         ]
         self.shell.find_line_magic("pdef")(arg, namespaces=namespaces)
 
     def do_pdoc(self, arg):
         """Print the docstring for an object.
 
         The debugger interface to %pdoc."""
         namespaces = [
             ("Locals", self.curframe_locals),
             ("Globals", self.curframe.f_globals),
         ]
         self.shell.find_line_magic("pdoc")(arg, namespaces=namespaces)
 
     def do_pfile(self, arg):
         """Print (or run through pager) the file where an object is defined.
 
         The debugger interface to %pfile.
         """
         namespaces = [
             ("Locals", self.curframe_locals),
             ("Globals", self.curframe.f_globals),
         ]
         self.shell.find_line_magic("pfile")(arg, namespaces=namespaces)
 
     def do_pinfo(self, arg):
         """Provide detailed information about an object.
 
         The debugger interface to %pinfo, i.e., obj?."""
         namespaces = [
             ("Locals", self.curframe_locals),
             ("Globals", self.curframe.f_globals),
         ]
         self.shell.find_line_magic("pinfo")(arg, namespaces=namespaces)
 
     def do_pinfo2(self, arg):
         """Provide extra detailed information about an object.
 
         The debugger interface to %pinfo2, i.e., obj??."""
         namespaces = [
             ("Locals", self.curframe_locals),
             ("Globals", self.curframe.f_globals),
         ]
         self.shell.find_line_magic("pinfo2")(arg, namespaces=namespaces)
 
     def do_psource(self, arg):
         """Print (or run through pager) the source code for an object."""
         namespaces = [
             ("Locals", self.curframe_locals),
             ("Globals", self.curframe.f_globals),
         ]
         self.shell.find_line_magic("psource")(arg, namespaces=namespaces)
 
     def do_where(self, arg):
         """w(here)
         Print a stack trace, with the most recent frame at the bottom.
         An arrow indicates the "current frame", which determines the
         context of most commands. 'bt' is an alias for this command.
 
         Take a number as argument as an (optional) number of context line to
         print"""
         if arg:
             try:
                 context = int(arg)
             except ValueError as err:
                 self.error(err)
                 return
             self.print_stack_trace(context)
         else:
             self.print_stack_trace()
 
     do_w = do_where
 
     def break_anywhere(self, frame):
         """
 
         _stop_in_decorator_internals is overly restrictive, as we may still want
         to trace function calls, so we need to also update break_anywhere so
         that is we don't `stop_here`, because of debugger skip, we may still
         stop at any point inside the function
 
         """
 
         sup = super().break_anywhere(frame)
         if sup:
             return sup
         if self._predicates["debuggerskip"]:
             if DEBUGGERSKIP in frame.f_code.co_varnames:
                 return True
             if frame.f_back and self._get_frame_locals(frame.f_back).get(DEBUGGERSKIP):
                 return True
         return False
 
-    @skip_doctest
     def _is_in_decorator_internal_and_should_skip(self, frame):
         """
         Utility to tell us whether we are in a decorator internal and should stop.
 
 
 
         """
 
         # if we are disabled don't skip
         if not self._predicates["debuggerskip"]:
             return False
 
         # if frame is tagged, skip by default.
         if DEBUGGERSKIP in frame.f_code.co_varnames:
             return True
 
         # if one of the parent frame value set to True skip as well.
 
         cframe = frame
         while getattr(cframe, "f_back", None):
             cframe = cframe.f_back
             if self._get_frame_locals(cframe).get(DEBUGGERSKIP):
                 return True
 
         return False
 
     def stop_here(self, frame):
 
         if self._is_in_decorator_internal_and_should_skip(frame) is True:
             return False
 
         hidden = False
         if self.skip_hidden:
             hidden = self._hidden_predicate(frame)
         if hidden:
             if self.report_skipped:
                 Colors = self.color_scheme_table.active_colors
                 ColorsNormal = Colors.Normal
                 print(
                     f"{Colors.excName}    [... skipped 1 hidden frame]{ColorsNormal}\n"
                 )
         return super().stop_here(frame)
 
     def do_up(self, arg):
         """u(p) [count]
         Move the current frame count (default one) levels up in the
         stack trace (to an older frame).
 
         Will skip hidden frames.
         """
         # modified version of upstream that skips
         # frames with __tracebackhide__
         if self.curindex == 0:
             self.error("Oldest frame")
             return
         try:
             count = int(arg or 1)
         except ValueError:
             self.error("Invalid frame count (%s)" % arg)
             return
         skipped = 0
         if count < 0:
             _newframe = 0
         else:
             counter = 0
             hidden_frames = self.hidden_frames(self.stack)
             for i in range(self.curindex - 1, -1, -1):
                 if hidden_frames[i] and self.skip_hidden:
                     skipped += 1
                     continue
                 counter += 1
                 if counter >= count:
                     break
             else:
                 # if no break occurred.
                 self.error(
                     "all frames above hidden, use `skip_hidden False` to get get into those."
                 )
                 return
 
             Colors = self.color_scheme_table.active_colors
             ColorsNormal = Colors.Normal
             _newframe = i
         self._select_frame(_newframe)
         if skipped:
             print(
                 f"{Colors.excName}    [... skipped {skipped} hidden frame(s)]{ColorsNormal}\n"
             )
 
     def do_down(self, arg):
         """d(own) [count]
         Move the current frame count (default one) levels down in the
         stack trace (to a newer frame).
 
         Will skip hidden frames.
         """
         if self.curindex + 1 == len(self.stack):
             self.error("Newest frame")
             return
         try:
             count = int(arg or 1)
         except ValueError:
             self.error("Invalid frame count (%s)" % arg)
             return
         if count < 0:
             _newframe = len(self.stack) - 1
         else:
             counter = 0
             skipped = 0
             hidden_frames = self.hidden_frames(self.stack)
             for i in range(self.curindex + 1, len(self.stack)):
                 if hidden_frames[i] and self.skip_hidden:
                     skipped += 1
                     continue
                 counter += 1
                 if counter >= count:
                     break
             else:
                 self.error(
                     "all frames below hidden, use `skip_hidden False` to get get into those."
                 )
                 return
 
             Colors = self.color_scheme_table.active_colors
             ColorsNormal = Colors.Normal
             if skipped:
                 print(
                     f"{Colors.excName}    [... skipped {skipped} hidden frame(s)]{ColorsNormal}\n"
                 )
             _newframe = i
 
         self._select_frame(_newframe)
 
     do_d = do_down
     do_u = do_up
 
     def do_context(self, context):
         """context number_of_lines
         Set the number of lines of source code to show when displaying
         stacktrace information.
         """
         try:
             new_context = int(context)
             if new_context <= 0:
                 raise ValueError()
             self.context = new_context
         except ValueError:
             self.error("The 'context' command requires a positive integer argument.")
 
 
 class InterruptiblePdb(Pdb):
     """Version of debugger where KeyboardInterrupt exits the debugger altogether."""
 
     def cmdloop(self, intro=None):
         """Wrap cmdloop() such that KeyboardInterrupt stops the debugger."""
         try:
             return OldPdb.cmdloop(self, intro=intro)
         except KeyboardInterrupt:
             self.stop_here = lambda frame: False
             self.do_quit("")
             sys.settrace(None)
             self.quitting = False
             raise
 
     def _cmdloop(self):
         while True:
             try:
                 # keyboard interrupts allow for an easy way to cancel
                 # the current command, so allow them during interactive input
                 self.allow_kbdint = True
                 self.cmdloop()
                 self.allow_kbdint = False
                 break
             except KeyboardInterrupt:
                 self.message('--KeyboardInterrupt--')
                 raise
 
 
 def set_trace(frame=None):
     """
     Start debugging from `frame`.
 
     If frame is not specified, debugging starts from caller's frame.
     """
     Pdb().set_trace(frame or sys._getframe().f_back)

IPython/core/magics/execution.py

@@ -1,1517 +1,1516 b''
 # -*- coding: utf-8 -*-
 """Implementation of execution-related magic functions."""
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 
 import ast
 import bdb
 import builtins as builtin_mod
 import gc
 import itertools
 import os
 import shlex
 import sys
 import time
 import timeit
 import math
 import re
 from pdb import Restart
 
 import cProfile as profile
 import pstats
 
 from IPython.core import oinspect
 from IPython.core import magic_arguments
 from IPython.core import page
 from IPython.core.error import UsageError
 from IPython.core.macro import Macro
 from IPython.core.magic import (Magics, magics_class, line_magic, cell_magic,
                                 line_cell_magic, on_off, needs_local_scope,
                                 no_var_expand)
 from IPython.testing.skipdoctest import skip_doctest
 from IPython.utils.contexts import preserve_keys
 from IPython.utils.capture import capture_output
 from IPython.utils.ipstruct import Struct
 from IPython.utils.module_paths import find_mod
 from IPython.utils.path import get_py_filename, shellglob
 from IPython.utils.timing import clock, clock2
 from warnings import warn
 from logging import error
 from pathlib import Path
 from io import StringIO
 from pathlib import Path
 
 if sys.version_info > (3,8):
     from ast import Module
 else :
     # mock the new API, ignore second argument
     # see https://github.com/ipython/ipython/issues/11590
     from ast import Module as OriginalModule
     Module = lambda nodelist, type_ignores: OriginalModule(nodelist)
 
 
 #-----------------------------------------------------------------------------
 # Magic implementation classes
 #-----------------------------------------------------------------------------
 
 
 class TimeitResult(object):
     """
     Object returned by the timeit magic with info about the run.
 
     Contains the following attributes :
 
     loops: (int) number of loops done per measurement
     repeat: (int) number of times the measurement has been repeated
     best: (float) best execution time / number
     all_runs: (list of float) execution time of each run (in s)
     compile_time: (float) time of statement compilation (s)
 
     """
     def __init__(self, loops, repeat, best, worst, all_runs, compile_time, precision):
         self.loops = loops
         self.repeat = repeat
         self.best = best
         self.worst = worst
         self.all_runs = all_runs
         self.compile_time = compile_time
         self._precision = precision
         self.timings = [ dt / self.loops for dt in all_runs]
 
     @property
     def average(self):
         return math.fsum(self.timings) / len(self.timings)
 
     @property
     def stdev(self):
         mean = self.average
         return (math.fsum([(x - mean) ** 2 for x in self.timings]) / len(self.timings)) ** 0.5
 
     def __str__(self):
         pm = '+-'
         if hasattr(sys.stdout, 'encoding') and sys.stdout.encoding:
             try:
                 u'\xb1'.encode(sys.stdout.encoding)
                 pm = u'\xb1'
             except:
                 pass
         return (
             u"{mean} {pm} {std} per loop (mean {pm} std. dev. of {runs} run{run_plural}, {loops} loop{loop_plural} each)"
                 .format(
                     pm = pm,
                     runs = self.repeat,
                     loops = self.loops,
                     loop_plural = "" if self.loops == 1 else "s",
                     run_plural = "" if self.repeat == 1 else "s",
                     mean = _format_time(self.average, self._precision),
                     std = _format_time(self.stdev, self._precision))
                 )
 
     def _repr_pretty_(self, p , cycle):
         unic = self.__str__()
         p.text(u'<TimeitResult : '+unic+u'>')
 
 
 class TimeitTemplateFiller(ast.NodeTransformer):
     """Fill in the AST template for timing execution.
 
     This is quite closely tied to the template definition, which is in
     :meth:`ExecutionMagics.timeit`.
     """
     def __init__(self, ast_setup, ast_stmt):
         self.ast_setup = ast_setup
         self.ast_stmt = ast_stmt
 
     def visit_FunctionDef(self, node):
         "Fill in the setup statement"
         self.generic_visit(node)
         if node.name == "inner":
             node.body[:1] = self.ast_setup.body
 
         return node
 
     def visit_For(self, node):
         "Fill in the statement to be timed"
         if getattr(getattr(node.body[0], 'value', None), 'id', None) == 'stmt':
             node.body = self.ast_stmt.body
         return node
 
 
 class Timer(timeit.Timer):
     """Timer class that explicitly uses self.inner
     
     which is an undocumented implementation detail of CPython,
     not shared by PyPy.
     """
     # Timer.timeit copied from CPython 3.4.2
     def timeit(self, number=timeit.default_number):
         """Time 'number' executions of the main statement.
 
         To be precise, this executes the setup statement once, and
         then returns the time it takes to execute the main statement
         a number of times, as a float measured in seconds.  The
         argument is the number of times through the loop, defaulting
         to one million.  The main statement, the setup statement and
         the timer function to be used are passed to the constructor.
         """
         it = itertools.repeat(None, number)
         gcold = gc.isenabled()
         gc.disable()
         try:
             timing = self.inner(it, self.timer)
         finally:
             if gcold:
                 gc.enable()
         return timing
 
 
 @magics_class
 class ExecutionMagics(Magics):
     """Magics related to code execution, debugging, profiling, etc.
 
     """
 
     def __init__(self, shell):
         super(ExecutionMagics, self).__init__(shell)
         # Default execution function used to actually run user code.
         self.default_runner = None
 
     @skip_doctest
     @no_var_expand
     @line_cell_magic
     def prun(self, parameter_s='', cell=None):
 
         """Run a statement through the python code profiler.
 
         Usage, in line mode:
           %prun [options] statement
 
         Usage, in cell mode:
           %%prun [options] [statement]
           code...
           code...
 
         In cell mode, the additional code lines are appended to the (possibly
         empty) statement in the first line.  Cell mode allows you to easily
         profile multiline blocks without having to put them in a separate
         function.
 
         The given statement (which doesn't require quote marks) is run via the
         python profiler in a manner similar to the profile.run() function.
         Namespaces are internally managed to work correctly; profile.run
         cannot be used in IPython because it makes certain assumptions about
         namespaces which do not hold under IPython.
 
         Options:
 
         -l <limit>
           you can place restrictions on what or how much of the
           profile gets printed. The limit value can be:
 
              * A string: only information for function names containing this string
                is printed.
 
              * An integer: only these many lines are printed.
 
              * A float (between 0 and 1): this fraction of the report is printed
                (for example, use a limit of 0.4 to see the topmost 40% only).
 
           You can combine several limits with repeated use of the option. For
           example, ``-l __init__ -l 5`` will print only the topmost 5 lines of
           information about class constructors.
 
         -r
           return the pstats.Stats object generated by the profiling. This
           object has all the information about the profile in it, and you can
           later use it for further analysis or in other functions.
 
         -s <key>
           sort profile by given key. You can provide more than one key
           by using the option several times: '-s key1 -s key2 -s key3...'. The
           default sorting key is 'time'.
 
           The following is copied verbatim from the profile documentation
           referenced below:
 
           When more than one key is provided, additional keys are used as
           secondary criteria when the there is equality in all keys selected
           before them.
 
           Abbreviations can be used for any key names, as long as the
           abbreviation is unambiguous.  The following are the keys currently
           defined:
 
           ============  =====================
           Valid Arg     Meaning
           ============  =====================
           "calls"       call count
           "cumulative"  cumulative time
           "file"        file name
           "module"      file name
           "pcalls"      primitive call count
           "line"        line number
           "name"        function name
           "nfl"         name/file/line
           "stdname"     standard name
           "time"        internal time
           ============  =====================
 
           Note that all sorts on statistics are in descending order (placing
           most time consuming items first), where as name, file, and line number
           searches are in ascending order (i.e., alphabetical). The subtle
           distinction between "nfl" and "stdname" is that the standard name is a
           sort of the name as printed, which means that the embedded line
           numbers get compared in an odd way.  For example, lines 3, 20, and 40
           would (if the file names were the same) appear in the string order
           "20" "3" and "40".  In contrast, "nfl" does a numeric compare of the
           line numbers.  In fact, sort_stats("nfl") is the same as
           sort_stats("name", "file", "line").
 
         -T <filename>
           save profile results as shown on screen to a text
           file. The profile is still shown on screen.
 
         -D <filename>
           save (via dump_stats) profile statistics to given
           filename. This data is in a format understood by the pstats module, and
           is generated by a call to the dump_stats() method of profile
           objects. The profile is still shown on screen.
 
         -q
           suppress output to the pager.  Best used with -T and/or -D above.
 
         If you want to run complete programs under the profiler's control, use
         ``%run -p [prof_opts] filename.py [args to program]`` where prof_opts
         contains profiler specific options as described here.
 
         You can read the complete documentation for the profile module with::
 
           In [1]: import profile; profile.help()
 
         .. versionchanged:: 7.3
             User variables are no longer expanded,
             the magic line is always left unmodified.
 
         """
         opts, arg_str = self.parse_options(parameter_s, 'D:l:rs:T:q',
                                            list_all=True, posix=False)
         if cell is not None:
             arg_str += '\n' + cell
         arg_str = self.shell.transform_cell(arg_str)
         return self._run_with_profiler(arg_str, opts, self.shell.user_ns)
 
     def _run_with_profiler(self, code, opts, namespace):
         """
         Run `code` with profiler.  Used by ``%prun`` and ``%run -p``.
 
         Parameters
         ----------
         code : str
             Code to be executed.
         opts : Struct
             Options parsed by `self.parse_options`.
         namespace : dict
             A dictionary for Python namespace (e.g., `self.shell.user_ns`).
 
         """
 
         # Fill default values for unspecified options:
         opts.merge(Struct(D=[''], l=[], s=['time'], T=['']))
 
         prof = profile.Profile()
         try:
             prof = prof.runctx(code, namespace, namespace)
             sys_exit = ''
         except SystemExit:
             sys_exit = """*** SystemExit exception caught in code being profiled."""
 
         stats = pstats.Stats(prof).strip_dirs().sort_stats(*opts.s)
 
         lims = opts.l
         if lims:
             lims = []  # rebuild lims with ints/floats/strings
             for lim in opts.l:
                 try:
                     lims.append(int(lim))
                 except ValueError:
                     try:
                         lims.append(float(lim))
                     except ValueError:
                         lims.append(lim)
 
         # Trap output.
         stdout_trap = StringIO()
         stats_stream = stats.stream
         try:
             stats.stream = stdout_trap
             stats.print_stats(*lims)
         finally:
             stats.stream = stats_stream
 
         output = stdout_trap.getvalue()
         output = output.rstrip()
 
         if 'q' not in opts:
             page.page(output)
         print(sys_exit, end=' ')
 
         dump_file = opts.D[0]
         text_file = opts.T[0]
         if dump_file:
             prof.dump_stats(dump_file)
             print(
                 f"\n*** Profile stats marshalled to file {repr(dump_file)}.{sys_exit}"
             )
         if text_file:
             pfile = Path(text_file)
             pfile.touch(exist_ok=True)
             pfile.write_text(output)
 
             print(
                 f"\n*** Profile printout saved to text file {repr(text_file)}.{sys_exit}"
             )
 
         if 'r' in opts:
             return stats
 
         return None
 
     @line_magic
     def pdb(self, parameter_s=''):
         """Control the automatic calling of the pdb interactive debugger.
 
         Call as '%pdb on', '%pdb 1', '%pdb off' or '%pdb 0'. If called without
         argument it works as a toggle.
 
         When an exception is triggered, IPython can optionally call the
         interactive pdb debugger after the traceback printout. %pdb toggles
         this feature on and off.
 
         The initial state of this feature is set in your configuration
         file (the option is ``InteractiveShell.pdb``).
 
         If you want to just activate the debugger AFTER an exception has fired,
         without having to type '%pdb on' and rerunning your code, you can use
         the %debug magic."""
 
         par = parameter_s.strip().lower()
 
         if par:
             try:
                 new_pdb = {'off':0,'0':0,'on':1,'1':1}[par]
             except KeyError:
                 print ('Incorrect argument. Use on/1, off/0, '
                        'or nothing for a toggle.')
                 return
         else:
             # toggle
             new_pdb = not self.shell.call_pdb
 
         # set on the shell
         self.shell.call_pdb = new_pdb
         print('Automatic pdb calling has been turned',on_off(new_pdb))
 
-    @skip_doctest
     @magic_arguments.magic_arguments()
     @magic_arguments.argument('--breakpoint', '-b', metavar='FILE:LINE',
         help="""
         Set break point at LINE in FILE.
         """
     )
     @magic_arguments.argument('statement', nargs='*',
         help="""
         Code to run in debugger.
         You can omit this in cell magic mode.
         """
     )
     @no_var_expand
     @line_cell_magic
     def debug(self, line='', cell=None):
         """Activate the interactive debugger.
 
         This magic command support two ways of activating debugger.
         One is to activate debugger before executing code.  This way, you
         can set a break point, to step through the code from the point.
         You can use this mode by giving statements to execute and optionally
         a breakpoint.
 
         The other one is to activate debugger in post-mortem mode.  You can
         activate this mode simply running %debug without any argument.
         If an exception has just occurred, this lets you inspect its stack
         frames interactively.  Note that this will always work only on the last
         traceback that occurred, so you must call this quickly after an
         exception that you wish to inspect has fired, because if another one
         occurs, it clobbers the previous one.
 
         If you want IPython to automatically do this on every exception, see
         the %pdb magic for more details.
 
         .. versionchanged:: 7.3
             When running code, user variables are no longer expanded,
             the magic line is always left unmodified.
 
         """
         args = magic_arguments.parse_argstring(self.debug, line)
 
         if not (args.breakpoint or args.statement or cell):
             self._debug_post_mortem()
         elif not (args.breakpoint or cell):
             # If there is no breakpoints, the line is just code to execute
             self._debug_exec(line, None)
         else:
             # Here we try to reconstruct the code from the output of
             # parse_argstring. This might not work if the code has spaces
             # For example this fails for `print("a b")`
             code = "\n".join(args.statement)
             if cell:
                 code += "\n" + cell
             self._debug_exec(code, args.breakpoint)
 
     def _debug_post_mortem(self):
         self.shell.debugger(force=True)
 
     def _debug_exec(self, code, breakpoint):
         if breakpoint:
             (filename, bp_line) = breakpoint.rsplit(':', 1)
             bp_line = int(bp_line)
         else:
             (filename, bp_line) = (None, None)
         self._run_with_debugger(code, self.shell.user_ns, filename, bp_line)
 
     @line_magic
     def tb(self, s):
         """Print the last traceback.
 
         Optionally, specify an exception reporting mode, tuning the
         verbosity of the traceback. By default the currently-active exception
         mode is used. See %xmode for changing exception reporting modes.
 
         Valid modes: Plain, Context, Verbose, and Minimal.
         """
         interactive_tb = self.shell.InteractiveTB
         if s:
             # Switch exception reporting mode for this one call.
             # Ensure it is switched back.
             def xmode_switch_err(name):
                 warn('Error changing %s exception modes.\n%s' %
                     (name,sys.exc_info()[1]))
 
             new_mode = s.strip().capitalize()
             original_mode = interactive_tb.mode
             try:
                 try:
                     interactive_tb.set_mode(mode=new_mode)
                 except Exception:
                     xmode_switch_err('user')
                 else:
                     self.shell.showtraceback()
             finally:
                 interactive_tb.set_mode(mode=original_mode)
         else:
             self.shell.showtraceback()
 
     @skip_doctest
     @line_magic
     def run(self, parameter_s='', runner=None,
                   file_finder=get_py_filename):
         """Run the named file inside IPython as a program.
 
         Usage::
         
           %run [-n -i -e -G]
                [( -t [-N<N>] | -d [-b<N>] | -p [profile options] )]
                ( -m mod | filename ) [args]
 
         The filename argument should be either a pure Python script (with
         extension ``.py``), or a file with custom IPython syntax (such as
         magics). If the latter, the file can be either a script with ``.ipy``
         extension, or a Jupyter notebook with ``.ipynb`` extension. When running
         a Jupyter notebook, the output from print statements and other
         displayed objects will appear in the terminal (even matplotlib figures
         will open, if a terminal-compliant backend is being used). Note that,
         at the system command line, the ``jupyter run`` command offers similar
         functionality for executing notebooks (albeit currently with some
         differences in supported options).
 
         Parameters after the filename are passed as command-line arguments to
         the program (put in sys.argv). Then, control returns to IPython's
         prompt.
 
         This is similar to running at a system prompt ``python file args``,
         but with the advantage of giving you IPython's tracebacks, and of
         loading all variables into your interactive namespace for further use
         (unless -p is used, see below).
 
         The file is executed in a namespace initially consisting only of
         ``__name__=='__main__'`` and sys.argv constructed as indicated. It thus
         sees its environment as if it were being run as a stand-alone program
         (except for sharing global objects such as previously imported
         modules). But after execution, the IPython interactive namespace gets
         updated with all variables defined in the program (except for __name__
         and sys.argv). This allows for very convenient loading of code for
         interactive work, while giving each program a 'clean sheet' to run in.
 
         Arguments are expanded using shell-like glob match.  Patterns
         '*', '?', '[seq]' and '[!seq]' can be used.  Additionally,
         tilde '~' will be expanded into user's home directory.  Unlike
         real shells, quotation does not suppress expansions.  Use
         *two* back slashes (e.g. ``\\\\*``) to suppress expansions.
         To completely disable these expansions, you can use -G flag.
 
         On Windows systems, the use of single quotes `'` when specifying 
         a file is not supported. Use double quotes `"`.
 
         Options:
 
         -n
           __name__ is NOT set to '__main__', but to the running file's name
           without extension (as python does under import).  This allows running
           scripts and reloading the definitions in them without calling code
           protected by an ``if __name__ == "__main__"`` clause.
 
         -i
           run the file in IPython's namespace instead of an empty one. This
           is useful if you are experimenting with code written in a text editor
           which depends on variables defined interactively.
 
         -e
           ignore sys.exit() calls or SystemExit exceptions in the script
           being run.  This is particularly useful if IPython is being used to
           run unittests, which always exit with a sys.exit() call.  In such
           cases you are interested in the output of the test results, not in
           seeing a traceback of the unittest module.
 
         -t
           print timing information at the end of the run.  IPython will give
           you an estimated CPU time consumption for your script, which under
           Unix uses the resource module to avoid the wraparound problems of
           time.clock().  Under Unix, an estimate of time spent on system tasks
           is also given (for Windows platforms this is reported as 0.0).
 
         If -t is given, an additional ``-N<N>`` option can be given, where <N>
         must be an integer indicating how many times you want the script to
         run.  The final timing report will include total and per run results.
 
         For example (testing the script uniq_stable.py)::
 
             In [1]: run -t uniq_stable
 
             IPython CPU timings (estimated):
               User  :    0.19597 s.
               System:        0.0 s.
 
             In [2]: run -t -N5 uniq_stable
 
             IPython CPU timings (estimated):
             Total runs performed: 5
               Times :      Total       Per run
               User  :   0.910862 s,  0.1821724 s.
               System:        0.0 s,        0.0 s.
 
         -d
           run your program under the control of pdb, the Python debugger.
           This allows you to execute your program step by step, watch variables,
           etc.  Internally, what IPython does is similar to calling::
 
               pdb.run('execfile("YOURFILENAME")')
 
           with a breakpoint set on line 1 of your file.  You can change the line
           number for this automatic breakpoint to be <N> by using the -bN option
           (where N must be an integer). For example::
 
               %run -d -b40 myscript
 
           will set the first breakpoint at line 40 in myscript.py.  Note that
           the first breakpoint must be set on a line which actually does
           something (not a comment or docstring) for it to stop execution.
 
           Or you can specify a breakpoint in a different file::
 
               %run -d -b myotherfile.py:20 myscript
 
           When the pdb debugger starts, you will see a (Pdb) prompt.  You must
           first enter 'c' (without quotes) to start execution up to the first
           breakpoint.
 
           Entering 'help' gives information about the use of the debugger.  You
           can easily see pdb's full documentation with "import pdb;pdb.help()"
           at a prompt.
 
         -p
           run program under the control of the Python profiler module (which
           prints a detailed report of execution times, function calls, etc).
 
           You can pass other options after -p which affect the behavior of the
           profiler itself. See the docs for %prun for details.
 
           In this mode, the program's variables do NOT propagate back to the
           IPython interactive namespace (because they remain in the namespace
           where the profiler executes them).
 
           Internally this triggers a call to %prun, see its documentation for
           details on the options available specifically for profiling.
 
         There is one special usage for which the text above doesn't apply:
         if the filename ends with .ipy[nb], the file is run as ipython script,
         just as if the commands were written on IPython prompt.
 
         -m
           specify module name to load instead of script path. Similar to
           the -m option for the python interpreter. Use this option last if you
           want to combine with other %run options. Unlike the python interpreter
           only source modules are allowed no .pyc or .pyo files.
           For example::
 
               %run -m example
 
           will run the example module.
 
         -G
           disable shell-like glob expansion of arguments.
 
         """
 
         # Logic to handle issue #3664
         # Add '--' after '-m <module_name>' to ignore additional args passed to a module.
         if '-m' in parameter_s and '--' not in parameter_s:
             argv = shlex.split(parameter_s, posix=(os.name == 'posix'))
             for idx, arg in enumerate(argv):
                 if arg and arg.startswith('-') and arg != '-':
                     if arg == '-m':
                         argv.insert(idx + 2, '--')
                         break
                 else:
                     # Positional arg, break
                     break
             parameter_s = ' '.join(shlex.quote(arg) for arg in argv)
 
         # get arguments and set sys.argv for program to be run.
         opts, arg_lst = self.parse_options(parameter_s,
                                            'nidtN:b:pD:l:rs:T:em:G',
                                            mode='list', list_all=1)
         if "m" in opts:
             modulename = opts["m"][0]
             modpath = find_mod(modulename)
             if modpath is None:
                 msg = '%r is not a valid modulename on sys.path'%modulename
                 raise Exception(msg)
             arg_lst = [modpath] + arg_lst
         try:
             fpath = None # initialize to make sure fpath is in scope later
             fpath = arg_lst[0]
             filename = file_finder(fpath)
         except IndexError as e:
             msg = 'you must provide at least a filename.'
             raise Exception(msg) from e
         except IOError as e:
             try:
                 msg = str(e)
             except UnicodeError:
                 msg = e.message
             if os.name == 'nt' and re.match(r"^'.*'$",fpath):
                 warn('For Windows, use double quotes to wrap a filename: %run "mypath\\myfile.py"')
             raise Exception(msg) from e
         except TypeError:
             if fpath in sys.meta_path:
                 filename = ""
             else:
                 raise
 
         if filename.lower().endswith(('.ipy', '.ipynb')):
             with preserve_keys(self.shell.user_ns, '__file__'):
                 self.shell.user_ns['__file__'] = filename
                 self.shell.safe_execfile_ipy(filename, raise_exceptions=True)
             return
 
         # Control the response to exit() calls made by the script being run
         exit_ignore = 'e' in opts
 
         # Make sure that the running script gets a proper sys.argv as if it
         # were run from a system shell.
         save_argv = sys.argv # save it for later restoring
 
         if 'G' in opts:
             args = arg_lst[1:]
         else:
             # tilde and glob expansion
             args = shellglob(map(os.path.expanduser,  arg_lst[1:]))
 
         sys.argv = [filename] + args  # put in the proper filename
 
         if 'n' in opts:
             name = Path(filename).stem
         else:
             name = '__main__'
 
         if 'i' in opts:
             # Run in user's interactive namespace
             prog_ns = self.shell.user_ns
             __name__save = self.shell.user_ns['__name__']
             prog_ns['__name__'] = name
             main_mod = self.shell.user_module
 
             # Since '%run foo' emulates 'python foo.py' at the cmd line, we must
             # set the __file__ global in the script's namespace
             # TK: Is this necessary in interactive mode?
             prog_ns['__file__'] = filename
         else:
             # Run in a fresh, empty namespace
 
             # The shell MUST hold a reference to prog_ns so after %run
             # exits, the python deletion mechanism doesn't zero it out
             # (leaving dangling references). See interactiveshell for details
             main_mod = self.shell.new_main_mod(filename, name)
             prog_ns = main_mod.__dict__
 
         # pickle fix.  See interactiveshell for an explanation.  But we need to
         # make sure that, if we overwrite __main__, we replace it at the end
         main_mod_name = prog_ns['__name__']
 
         if main_mod_name == '__main__':
             restore_main = sys.modules['__main__']
         else:
             restore_main = False
 
         # This needs to be undone at the end to prevent holding references to
         # every single object ever created.
         sys.modules[main_mod_name] = main_mod
 
         if 'p' in opts or 'd' in opts:
             if 'm' in opts:
                 code = 'run_module(modulename, prog_ns)'
                 code_ns = {
                     'run_module': self.shell.safe_run_module,
                     'prog_ns': prog_ns,
                     'modulename': modulename,
                 }
             else:
                 if 'd' in opts:
                     # allow exceptions to raise in debug mode
                     code = 'execfile(filename, prog_ns, raise_exceptions=True)'
                 else:
                     code = 'execfile(filename, prog_ns)'
                 code_ns = {
                     'execfile': self.shell.safe_execfile,
                     'prog_ns': prog_ns,
                     'filename': get_py_filename(filename),
                 }
 
         try:
             stats = None
             if 'p' in opts:
                 stats = self._run_with_profiler(code, opts, code_ns)
             else:
                 if 'd' in opts:
                     bp_file, bp_line = parse_breakpoint(
                         opts.get('b', ['1'])[0], filename)
                     self._run_with_debugger(
                         code, code_ns, filename, bp_line, bp_file)
                 else:
                     if 'm' in opts:
                         def run():
                             self.shell.safe_run_module(modulename, prog_ns)
                     else:
                         if runner is None:
                             runner = self.default_runner
                         if runner is None:
                             runner = self.shell.safe_execfile
 
                         def run():
                             runner(filename, prog_ns, prog_ns,
                                     exit_ignore=exit_ignore)
 
                     if 't' in opts:
                         # timed execution
                         try:
                             nruns = int(opts['N'][0])
                             if nruns < 1:
                                 error('Number of runs must be >=1')
                                 return
                         except (KeyError):
                             nruns = 1
                         self._run_with_timing(run, nruns)
                     else:
                         # regular execution
                         run()
 
             if 'i' in opts:
                 self.shell.user_ns['__name__'] = __name__save
             else:
                 # update IPython interactive namespace
 
                 # Some forms of read errors on the file may mean the
                 # __name__ key was never set; using pop we don't have to
                 # worry about a possible KeyError.
                 prog_ns.pop('__name__', None)
 
                 with preserve_keys(self.shell.user_ns, '__file__'):
                     self.shell.user_ns.update(prog_ns)
         finally:
             # It's a bit of a mystery why, but __builtins__ can change from
             # being a module to becoming a dict missing some key data after
             # %run.  As best I can see, this is NOT something IPython is doing
             # at all, and similar problems have been reported before:
             # http://coding.derkeiler.com/Archive/Python/comp.lang.python/2004-10/0188.html
             # Since this seems to be done by the interpreter itself, the best
             # we can do is to at least restore __builtins__ for the user on
             # exit.
             self.shell.user_ns['__builtins__'] = builtin_mod
 
             # Ensure key global structures are restored
             sys.argv = save_argv
             if restore_main:
                 sys.modules['__main__'] = restore_main
                 if '__mp_main__' in sys.modules:
                     sys.modules['__mp_main__'] = restore_main
             else:
                 # Remove from sys.modules the reference to main_mod we'd
                 # added.  Otherwise it will trap references to objects
                 # contained therein.
                 del sys.modules[main_mod_name]
 
         return stats
 
     def _run_with_debugger(self, code, code_ns, filename=None,
                            bp_line=None, bp_file=None):
         """
         Run `code` in debugger with a break point.
 
         Parameters
         ----------
         code : str
             Code to execute.
         code_ns : dict
             A namespace in which `code` is executed.
         filename : str
             `code` is ran as if it is in `filename`.
         bp_line : int, optional
             Line number of the break point.
         bp_file : str, optional
             Path to the file in which break point is specified.
             `filename` is used if not given.
 
         Raises
         ------
         UsageError
             If the break point given by `bp_line` is not valid.
 
         """
         deb = self.shell.InteractiveTB.pdb
         if not deb:
             self.shell.InteractiveTB.pdb = self.shell.InteractiveTB.debugger_cls()
             deb = self.shell.InteractiveTB.pdb
 
         # deb.checkline() fails if deb.curframe exists but is None; it can
         # handle it not existing. https://github.com/ipython/ipython/issues/10028
         if hasattr(deb, 'curframe'):
             del deb.curframe
 
         # reset Breakpoint state, which is moronically kept
         # in a class
         bdb.Breakpoint.next = 1
         bdb.Breakpoint.bplist = {}
         bdb.Breakpoint.bpbynumber = [None]
         deb.clear_all_breaks()
         if bp_line is not None:
             # Set an initial breakpoint to stop execution
             maxtries = 10
             bp_file = bp_file or filename
             checkline = deb.checkline(bp_file, bp_line)
             if not checkline:
                 for bp in range(bp_line + 1, bp_line + maxtries + 1):
                     if deb.checkline(bp_file, bp):
                         break
                 else:
                     msg = ("\nI failed to find a valid line to set "
                            "a breakpoint\n"
                            "after trying up to line: %s.\n"
                            "Please set a valid breakpoint manually "
                            "with the -b option." % bp)
                     raise UsageError(msg)
             # if we find a good linenumber, set the breakpoint
             deb.do_break('%s:%s' % (bp_file, bp_line))
 
         if filename:
             # Mimic Pdb._runscript(...)
             deb._wait_for_mainpyfile = True
             deb.mainpyfile = deb.canonic(filename)
 
         # Start file run
         print("NOTE: Enter 'c' at the %s prompt to continue execution." % deb.prompt)
         try:
             if filename:
                 # save filename so it can be used by methods on the deb object
                 deb._exec_filename = filename
             while True:
                 try:
                     trace = sys.gettrace()
                     deb.run(code, code_ns)
                 except Restart:
                     print("Restarting")
                     if filename:
                         deb._wait_for_mainpyfile = True
                         deb.mainpyfile = deb.canonic(filename)
                     continue
                 else:
                     break
                 finally:
                     sys.settrace(trace)
             
 
         except:
             etype, value, tb = sys.exc_info()
             # Skip three frames in the traceback: the %run one,
             # one inside bdb.py, and the command-line typed by the
             # user (run by exec in pdb itself).
             self.shell.InteractiveTB(etype, value, tb, tb_offset=3)
 
     @staticmethod
     def _run_with_timing(run, nruns):
         """
         Run function `run` and print timing information.
 
         Parameters
         ----------
         run : callable
             Any callable object which takes no argument.
         nruns : int
             Number of times to execute `run`.
 
         """
         twall0 = time.perf_counter()
         if nruns == 1:
             t0 = clock2()
             run()
             t1 = clock2()
             t_usr = t1[0] - t0[0]
             t_sys = t1[1] - t0[1]
             print("\nIPython CPU timings (estimated):")
             print("  User   : %10.2f s." % t_usr)
             print("  System : %10.2f s." % t_sys)
         else:
             runs = range(nruns)
             t0 = clock2()
             for nr in runs:
                 run()
             t1 = clock2()
             t_usr = t1[0] - t0[0]
             t_sys = t1[1] - t0[1]
             print("\nIPython CPU timings (estimated):")
             print("Total runs performed:", nruns)
             print("  Times  : %10s   %10s" % ('Total', 'Per run'))
             print("  User   : %10.2f s, %10.2f s." % (t_usr, t_usr / nruns))
             print("  System : %10.2f s, %10.2f s." % (t_sys, t_sys / nruns))
         twall1 = time.perf_counter()
         print("Wall time: %10.2f s." % (twall1 - twall0))
 
     @skip_doctest
     @no_var_expand
     @line_cell_magic
     @needs_local_scope
     def timeit(self, line='', cell=None, local_ns=None):
         """Time execution of a Python statement or expression
 
         Usage, in line mode:
           %timeit [-n<N> -r<R> [-t|-c] -q -p<P> -o] statement
         or in cell mode:
           %%timeit [-n<N> -r<R> [-t|-c] -q -p<P> -o] setup_code
           code
           code...
 
         Time execution of a Python statement or expression using the timeit
         module.  This function can be used both as a line and cell magic:
 
         - In line mode you can time a single-line statement (though multiple
           ones can be chained with using semicolons).
 
         - In cell mode, the statement in the first line is used as setup code
           (executed but not timed) and the body of the cell is timed.  The cell
           body has access to any variables created in the setup code.
 
         Options:
         -n<N>: execute the given statement <N> times in a loop. If <N> is not
         provided, <N> is determined so as to get sufficient accuracy.
 
         -r<R>: number of repeats <R>, each consisting of <N> loops, and take the
         best result.
         Default: 7
 
         -t: use time.time to measure the time, which is the default on Unix.
         This function measures wall time.
 
         -c: use time.clock to measure the time, which is the default on
         Windows and measures wall time. On Unix, resource.getrusage is used
         instead and returns the CPU user time.
 
         -p<P>: use a precision of <P> digits to display the timing result.
         Default: 3
 
         -q: Quiet, do not print result.
 
         -o: return a TimeitResult that can be stored in a variable to inspect
             the result in more details.
 
         .. versionchanged:: 7.3
             User variables are no longer expanded,
             the magic line is always left unmodified.
 
         Examples
         --------
         ::
 
           In [1]: %timeit pass
           8.26 ns ± 0.12 ns per loop (mean ± std. dev. of 7 runs, 100000000 loops each)
 
           In [2]: u = None
 
           In [3]: %timeit u is None
           29.9 ns ± 0.643 ns per loop (mean ± std. dev. of 7 runs, 10000000 loops each)
 
           In [4]: %timeit -r 4 u == None
 
           In [5]: import time
 
           In [6]: %timeit -n1 time.sleep(2)
 
 
         The times reported by %timeit will be slightly higher than those
         reported by the timeit.py script when variables are accessed. This is
         due to the fact that %timeit executes the statement in the namespace
         of the shell, compared with timeit.py, which uses a single setup
         statement to import function or create variables. Generally, the bias
         does not matter as long as results from timeit.py are not mixed with
         those from %timeit."""
 
         opts, stmt = self.parse_options(
             line, "n:r:tcp:qo", posix=False, strict=False, preserve_non_opts=True
         )
         if stmt == "" and cell is None:
             return
         
         timefunc = timeit.default_timer
         number = int(getattr(opts, "n", 0))
         default_repeat = 7 if timeit.default_repeat < 7 else timeit.default_repeat
         repeat = int(getattr(opts, "r", default_repeat))
         precision = int(getattr(opts, "p", 3))
         quiet = 'q' in opts
         return_result = 'o' in opts
         if hasattr(opts, "t"):
             timefunc = time.time
         if hasattr(opts, "c"):
             timefunc = clock
 
         timer = Timer(timer=timefunc)
         # this code has tight coupling to the inner workings of timeit.Timer,
         # but is there a better way to achieve that the code stmt has access
         # to the shell namespace?
         transform  = self.shell.transform_cell
 
         if cell is None:
             # called as line magic
             ast_setup = self.shell.compile.ast_parse("pass")
             ast_stmt = self.shell.compile.ast_parse(transform(stmt))
         else:
             ast_setup = self.shell.compile.ast_parse(transform(stmt))
             ast_stmt = self.shell.compile.ast_parse(transform(cell))
 
         ast_setup = self.shell.transform_ast(ast_setup)
         ast_stmt = self.shell.transform_ast(ast_stmt)
 
         # Check that these compile to valid Python code *outside* the timer func
         # Invalid code may become valid when put inside the function & loop,
         # which messes up error messages.
         # https://github.com/ipython/ipython/issues/10636
         self.shell.compile(ast_setup, "<magic-timeit-setup>", "exec")
         self.shell.compile(ast_stmt, "<magic-timeit-stmt>", "exec")
 
         # This codestring is taken from timeit.template - we fill it in as an
         # AST, so that we can apply our AST transformations to the user code
         # without affecting the timing code.
         timeit_ast_template = ast.parse('def inner(_it, _timer):\n'
                                         '    setup\n'
                                         '    _t0 = _timer()\n'
                                         '    for _i in _it:\n'
                                         '        stmt\n'
                                         '    _t1 = _timer()\n'
                                         '    return _t1 - _t0\n')
 
         timeit_ast = TimeitTemplateFiller(ast_setup, ast_stmt).visit(timeit_ast_template)
         timeit_ast = ast.fix_missing_locations(timeit_ast)
 
         # Track compilation time so it can be reported if too long
         # Minimum time above which compilation time will be reported
         tc_min = 0.1
 
         t0 = clock()
         code = self.shell.compile(timeit_ast, "<magic-timeit>", "exec")
         tc = clock()-t0
 
         ns = {}
         glob = self.shell.user_ns
         # handles global vars with same name as local vars. We store them in conflict_globs.
         conflict_globs = {}
         if local_ns and cell is None:
             for var_name, var_val in glob.items():
                 if var_name in local_ns:
                     conflict_globs[var_name] = var_val
             glob.update(local_ns)
             
         exec(code, glob, ns)
         timer.inner = ns["inner"]
 
         # This is used to check if there is a huge difference between the
         # best and worst timings.
         # Issue: https://github.com/ipython/ipython/issues/6471
         if number == 0:
             # determine number so that 0.2 <= total time < 2.0
             for index in range(0, 10):
                 number = 10 ** index
                 time_number = timer.timeit(number)
                 if time_number >= 0.2:
                     break
 
         all_runs = timer.repeat(repeat, number)
         best = min(all_runs) / number
         worst = max(all_runs) / number
         timeit_result = TimeitResult(number, repeat, best, worst, all_runs, tc, precision)
 
         # Restore global vars from conflict_globs
         if conflict_globs:
            glob.update(conflict_globs)
                 
         if not quiet :
             # Check best timing is greater than zero to avoid a
             # ZeroDivisionError.
             # In cases where the slowest timing is lesser than a microsecond
             # we assume that it does not really matter if the fastest
             # timing is 4 times faster than the slowest timing or not.
             if worst > 4 * best and best > 0 and worst > 1e-6:
                 print("The slowest run took %0.2f times longer than the "
                       "fastest. This could mean that an intermediate result "
                       "is being cached." % (worst / best))
            
             print( timeit_result )
 
             if tc > tc_min:
                 print("Compiler time: %.2f s" % tc)
         if return_result:
             return timeit_result
 
     @skip_doctest
     @no_var_expand
     @needs_local_scope
     @line_cell_magic
     def time(self,line='', cell=None, local_ns=None):
         """Time execution of a Python statement or expression.
 
         The CPU and wall clock times are printed, and the value of the
         expression (if any) is returned.  Note that under Win32, system time
         is always reported as 0, since it can not be measured.
         
         This function can be used both as a line and cell magic:
 
         - In line mode you can time a single-line statement (though multiple
           ones can be chained with using semicolons).
 
         - In cell mode, you can time the cell body (a directly
           following statement raises an error).
 
         This function provides very basic timing functionality.  Use the timeit
         magic for more control over the measurement.
 
         .. versionchanged:: 7.3
             User variables are no longer expanded,
             the magic line is always left unmodified.
 
         Examples
         --------
         ::
 
           In [1]: %time 2**128
           CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
           Wall time: 0.00
           Out[1]: 340282366920938463463374607431768211456L
 
           In [2]: n = 1000000
 
           In [3]: %time sum(range(n))
           CPU times: user 1.20 s, sys: 0.05 s, total: 1.25 s
           Wall time: 1.37
           Out[3]: 499999500000L
 
           In [4]: %time print 'hello world'
           hello world
           CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
           Wall time: 0.00
 
 
         .. note::
             The time needed by Python to compile the given expression will be
             reported if it is more than 0.1s.
 
             In the example below, the actual exponentiation is done by Python
             at compilation time, so while the expression can take a noticeable
             amount of time to compute, that time is purely due to the
             compilation::
 
                 In [5]: %time 3**9999;
                 CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
                 Wall time: 0.00 s
 
                 In [6]: %time 3**999999;
                 CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
                 Wall time: 0.00 s
                 Compiler : 0.78 s
         """
         # fail immediately if the given expression can't be compiled
         
         if line and cell:
             raise UsageError("Can't use statement directly after '%%time'!")
         
         if cell:
             expr = self.shell.transform_cell(cell)
         else:
             expr = self.shell.transform_cell(line)
 
         # Minimum time above which parse time will be reported
         tp_min = 0.1
 
         t0 = clock()
         expr_ast = self.shell.compile.ast_parse(expr)
         tp = clock()-t0
 
         # Apply AST transformations
         expr_ast = self.shell.transform_ast(expr_ast)
 
         # Minimum time above which compilation time will be reported
         tc_min = 0.1
 
         expr_val=None
         if len(expr_ast.body)==1 and isinstance(expr_ast.body[0], ast.Expr):
             mode = 'eval'
             source = '<timed eval>'
             expr_ast = ast.Expression(expr_ast.body[0].value)
         else:
             mode = 'exec'
             source = '<timed exec>'
             # multi-line %%time case
             if len(expr_ast.body) > 1 and isinstance(expr_ast.body[-1], ast.Expr):
                 expr_val= expr_ast.body[-1]
                 expr_ast = expr_ast.body[:-1]
                 expr_ast = Module(expr_ast, [])
                 expr_val = ast.Expression(expr_val.value)
 
         t0 = clock()
         code = self.shell.compile(expr_ast, source, mode)
         tc = clock()-t0
 
         # skew measurement as little as possible
         glob = self.shell.user_ns
         wtime = time.time
         # time execution
         wall_st = wtime()
         if mode=='eval':
             st = clock2()
             try:
                 out = eval(code, glob, local_ns)
             except:
                 self.shell.showtraceback()
                 return
             end = clock2()
         else:
             st = clock2()
             try:
                 exec(code, glob, local_ns)
                 out=None
                 # multi-line %%time case
                 if expr_val is not None:
                     code_2 = self.shell.compile(expr_val, source, 'eval')
                     out = eval(code_2, glob, local_ns)
             except:
                 self.shell.showtraceback()
                 return
             end = clock2()
 
         wall_end = wtime()
         # Compute actual times and report
         wall_time = wall_end-wall_st
         cpu_user = end[0]-st[0]
         cpu_sys = end[1]-st[1]
         cpu_tot = cpu_user+cpu_sys
         # On windows cpu_sys is always zero, so no new information to the next print 
         if sys.platform != 'win32':
             print("CPU times: user %s, sys: %s, total: %s" % \
                 (_format_time(cpu_user),_format_time(cpu_sys),_format_time(cpu_tot)))
         print("Wall time: %s" % _format_time(wall_time))
         if tc > tc_min:
             print("Compiler : %s" % _format_time(tc))
         if tp > tp_min:
             print("Parser   : %s" % _format_time(tp))
         return out
 
     @skip_doctest
     @line_magic
     def macro(self, parameter_s=''):
         """Define a macro for future re-execution. It accepts ranges of history,
         filenames or string objects.
 
         Usage:\\
           %macro [options] name n1-n2 n3-n4 ... n5 .. n6 ...
 
         Options:
 
           -r: use 'raw' input.  By default, the 'processed' history is used,
           so that magics are loaded in their transformed version to valid
           Python.  If this option is given, the raw input as typed at the
           command line is used instead.
           
           -q: quiet macro definition.  By default, a tag line is printed 
           to indicate the macro has been created, and then the contents of 
           the macro are printed.  If this option is given, then no printout
           is produced once the macro is created.
 
         This will define a global variable called `name` which is a string
         made of joining the slices and lines you specify (n1,n2,... numbers
         above) from your input history into a single string. This variable
         acts like an automatic function which re-executes those lines as if
         you had typed them. You just type 'name' at the prompt and the code
         executes.
 
         The syntax for indicating input ranges is described in %history.
 
         Note: as a 'hidden' feature, you can also use traditional python slice
         notation, where N:M means numbers N through M-1.
 
         For example, if your history contains (print using %hist -n )::
 
           44: x=1
           45: y=3
           46: z=x+y
           47: print x
           48: a=5
           49: print 'x',x,'y',y
 
         you can create a macro with lines 44 through 47 (included) and line 49
         called my_macro with::
 
           In [55]: %macro my_macro 44-47 49
 
         Now, typing `my_macro` (without quotes) will re-execute all this code
         in one pass.
 
         You don't need to give the line-numbers in order, and any given line
         number can appear multiple times. You can assemble macros with any
         lines from your input history in any order.
 
         The macro is a simple object which holds its value in an attribute,
         but IPython's display system checks for macros and executes them as
         code instead of printing them when you type their name.
 
         You can view a macro's contents by explicitly printing it with::
 
           print macro_name
 
         """
         opts,args = self.parse_options(parameter_s,'rq',mode='list')
         if not args:   # List existing macros
             return sorted(k for k,v in self.shell.user_ns.items() if isinstance(v, Macro))
         if len(args) == 1:
             raise UsageError(
                 "%macro insufficient args; usage '%macro name n1-n2 n3-4...")
         name, codefrom = args[0], " ".join(args[1:])
 
         #print 'rng',ranges  # dbg
         try:
             lines = self.shell.find_user_code(codefrom, 'r' in opts)
         except (ValueError, TypeError) as e:
             print(e.args[0])
             return
         macro = Macro(lines)
         self.shell.define_macro(name, macro)
         if not ( 'q' in opts) : 
             print('Macro `%s` created. To execute, type its name (without quotes).' % name)
             print('=== Macro contents: ===')
             print(macro, end=' ')
 
     @magic_arguments.magic_arguments()
     @magic_arguments.argument('output', type=str, default='', nargs='?',
         help="""The name of the variable in which to store output.
         This is a utils.io.CapturedIO object with stdout/err attributes
         for the text of the captured output.
 
         CapturedOutput also has a show() method for displaying the output,
         and __call__ as well, so you can use that to quickly display the
         output.
 
         If unspecified, captured output is discarded.
         """
     )
     @magic_arguments.argument('--no-stderr', action="store_true",
         help="""Don't capture stderr."""
     )
     @magic_arguments.argument('--no-stdout', action="store_true",
         help="""Don't capture stdout."""
     )
     @magic_arguments.argument('--no-display', action="store_true",
         help="""Don't capture IPython's rich display."""
     )
     @cell_magic
     def capture(self, line, cell):
         """run the cell, capturing stdout, stderr, and IPython's rich display() calls."""
         args = magic_arguments.parse_argstring(self.capture, line)
         out = not args.no_stdout
         err = not args.no_stderr
         disp = not args.no_display
         with capture_output(out, err, disp) as io:
             self.shell.run_cell(cell)
         if args.output:
             self.shell.user_ns[args.output] = io
 
 def parse_breakpoint(text, current_file):
     '''Returns (file, line) for file:line and (current_file, line) for line'''
     colon = text.find(':')
     if colon == -1:
         return current_file, int(text)
     else:
         return text[:colon], int(text[colon+1:])
     
 def _format_time(timespan, precision=3):
     """Formats the timespan in a human readable form"""
 
     if timespan >= 60.0:
         # we have more than a minute, format that in a human readable form
         # Idea from http://snipplr.com/view/5713/
         parts = [("d", 60*60*24),("h", 60*60),("min", 60), ("s", 1)]
         time = []
         leftover = timespan
         for suffix, length in parts:
             value = int(leftover / length)
             if value > 0:
                 leftover = leftover % length
                 time.append(u'%s%s' % (str(value), suffix))
             if leftover < 1:
                 break
         return " ".join(time)
 
     
     # Unfortunately the unicode 'micro' symbol can cause problems in
     # certain terminals.  
     # See bug: https://bugs.launchpad.net/ipython/+bug/348466
     # Try to prevent crashes by being more secure than it needs to
     # E.g. eclipse is able to print a µ, but has no sys.stdout.encoding set.
     units = [u"s", u"ms",u'us',"ns"] # the save value   
     if hasattr(sys.stdout, 'encoding') and sys.stdout.encoding:
         try:
             u'\xb5'.encode(sys.stdout.encoding)
             units = [u"s", u"ms",u'\xb5s',"ns"]
         except:
             pass
     scaling = [1, 1e3, 1e6, 1e9]
         
     if timespan > 0.0:
         order = min(-int(math.floor(math.log10(timespan)) // 3), 3)
     else:
         order = 3
     return u"%.*g %s" % (precision, timespan * scaling[order], units[order])

IPython/core/magics/osm.py

@@ -1,859 +1,856 b''
 """Implementation of magic functions for interaction with the OS.
 
 Note: this module is named 'osm' instead of 'os' to avoid a collision with the
 builtin.
 """
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 import io
 import os
 import re
 import sys
 from pprint import pformat
 
 from IPython.core import magic_arguments
 from IPython.core import oinspect
 from IPython.core import page
 from IPython.core.alias import AliasError, Alias
 from IPython.core.error import UsageError
 from IPython.core.magic import  (
     Magics, compress_dhist, magics_class, line_magic, cell_magic, line_cell_magic
 )
 from IPython.testing.skipdoctest import skip_doctest
 from IPython.utils.openpy import source_to_unicode
 from IPython.utils.process import abbrev_cwd
 from IPython.utils.terminal import set_term_title
 from traitlets import Bool
 from warnings import warn
 
 
 @magics_class
 class OSMagics(Magics):
     """Magics to interact with the underlying OS (shell-type functionality).
     """
 
     cd_force_quiet = Bool(False,
         help="Force %cd magic to be quiet even if -q is not passed."
     ).tag(config=True)
 
     def __init__(self, shell=None, **kwargs):
 
         # Now define isexec in a cross platform manner.
         self.is_posix = False
         self.execre = None
         if os.name == 'posix':
             self.is_posix = True
         else:
             try:
                 winext = os.environ['pathext'].replace(';','|').replace('.','')
             except KeyError:
                 winext = 'exe|com|bat|py'
             try:
                 self.execre = re.compile(r'(.*)\.(%s)$' % winext,re.IGNORECASE)
             except re.error:
                 warn("Seems like your pathext environmental "
                      "variable is malformed. Please check it to "
                      "enable a proper handle of file extensions "
                      "managed for your system")
                 winext = 'exe|com|bat|py'
                 self.execre = re.compile(r'(.*)\.(%s)$' % winext,re.IGNORECASE)
 
         # call up the chain
         super().__init__(shell=shell, **kwargs)
 
 
-    @skip_doctest
     def _isexec_POSIX(self, file):
         """
             Test for executable on a POSIX system
         """
         if os.access(file.path, os.X_OK):
             # will fail on maxOS if access is not X_OK
             return file.is_file()
         return False
 
 
     
-    @skip_doctest
     def _isexec_WIN(self, file):
         """
             Test for executable file on non POSIX system
         """
         return file.is_file() and self.execre.match(file.name) is not None
 
-    @skip_doctest
     def isexec(self, file):
         """
             Test for executable file on non POSIX system
         """
         if self.is_posix:
             return self._isexec_POSIX(file)
         else:
             return self._isexec_WIN(file)
 
 
     @skip_doctest
     @line_magic
     def alias(self, parameter_s=''):
         """Define an alias for a system command.
 
         '%alias alias_name cmd' defines 'alias_name' as an alias for 'cmd'
 
         Then, typing 'alias_name params' will execute the system command 'cmd
         params' (from your underlying operating system).
 
         Aliases have lower precedence than magic functions and Python normal
         variables, so if 'foo' is both a Python variable and an alias, the
         alias can not be executed until 'del foo' removes the Python variable.
 
         You can use the %l specifier in an alias definition to represent the
         whole line when the alias is called.  For example::
 
           In [2]: alias bracket echo "Input in brackets: <%l>"
           In [3]: bracket hello world
           Input in brackets: <hello world>
 
         You can also define aliases with parameters using %s specifiers (one
         per parameter)::
 
           In [1]: alias parts echo first %s second %s
           In [2]: %parts A B
           first A second B
           In [3]: %parts A
           Incorrect number of arguments: 2 expected.
           parts is an alias to: 'echo first %s second %s'
 
         Note that %l and %s are mutually exclusive.  You can only use one or
         the other in your aliases.
 
         Aliases expand Python variables just like system calls using ! or !!
         do: all expressions prefixed with '$' get expanded.  For details of
         the semantic rules, see PEP-215:
         http://www.python.org/peps/pep-0215.html.  This is the library used by
         IPython for variable expansion.  If you want to access a true shell
         variable, an extra $ is necessary to prevent its expansion by
         IPython::
 
           In [6]: alias show echo
           In [7]: PATH='A Python string'
           In [8]: show $PATH
           A Python string
           In [9]: show $$PATH
           /usr/local/lf9560/bin:/usr/local/intel/compiler70/ia32/bin:...
 
         You can use the alias facility to access all of $PATH.  See the %rehashx
         function, which automatically creates aliases for the contents of your
         $PATH.
 
         If called with no parameters, %alias prints the current alias table
         for your system.  For posix systems, the default aliases are 'cat',
         'cp', 'mv', 'rm', 'rmdir', and 'mkdir', and other platform-specific
         aliases are added.  For windows-based systems, the default aliases are
         'copy', 'ddir', 'echo', 'ls', 'ldir', 'mkdir', 'ren', and 'rmdir'.
 
         You can see the definition of alias by adding a question mark in the
         end::
 
           In [1]: cat?
           Repr: <alias cat for 'cat'>"""
 
         par = parameter_s.strip()
         if not par:
             aliases = sorted(self.shell.alias_manager.aliases)
             # stored = self.shell.db.get('stored_aliases', {} )
             # for k, v in stored:
             #     atab.append(k, v[0])
 
             print("Total number of aliases:", len(aliases))
             sys.stdout.flush()
             return aliases
 
         # Now try to define a new one
         try:
             alias,cmd = par.split(None, 1)
         except TypeError:
             print(oinspect.getdoc(self.alias))
             return
         
         try:
             self.shell.alias_manager.define_alias(alias, cmd)
         except AliasError as e:
             print(e)
     # end magic_alias
 
     @line_magic
     def unalias(self, parameter_s=''):
         """Remove an alias"""
 
         aname = parameter_s.strip()
         try:
             self.shell.alias_manager.undefine_alias(aname)
         except ValueError as e:
             print(e)
             return
         
         stored = self.shell.db.get('stored_aliases', {} )
         if aname in stored:
             print("Removing %stored alias",aname)
             del stored[aname]
             self.shell.db['stored_aliases'] = stored
 
     @line_magic
     def rehashx(self, parameter_s=''):
         """Update the alias table with all executable files in $PATH.
 
         rehashx explicitly checks that every entry in $PATH is a file
         with execute access (os.X_OK).
 
         Under Windows, it checks executability as a match against a
         '|'-separated string of extensions, stored in the IPython config
         variable win_exec_ext.  This defaults to 'exe|com|bat'.
 
         This function also resets the root module cache of module completer,
         used on slow filesystems.
         """
         from IPython.core.alias import InvalidAliasError
 
         # for the benefit of module completer in ipy_completers.py
         del self.shell.db['rootmodules_cache']
 
         path = [os.path.abspath(os.path.expanduser(p)) for p in
             os.environ.get('PATH','').split(os.pathsep)]
 
         syscmdlist = []
         savedir = os.getcwd()
 
         # Now walk the paths looking for executables to alias.
         try:
             # write the whole loop for posix/Windows so we don't have an if in
             # the innermost part
             if self.is_posix:
                 for pdir in path:
                     try:
                         os.chdir(pdir)
                     except OSError:
                         continue
 
                     # for python 3.6+ rewrite to: with os.scandir(pdir) as dirlist:
                     dirlist = os.scandir(path=pdir)
                     for ff in dirlist:
                         if self.isexec(ff):
                             fname = ff.name
                             try:
                                 # Removes dots from the name since ipython
                                 # will assume names with dots to be python.
                                 if not self.shell.alias_manager.is_alias(fname):
                                     self.shell.alias_manager.define_alias(
                                         fname.replace('.',''), fname)
                             except InvalidAliasError:
                                 pass
                             else:
                                 syscmdlist.append(fname)
             else:
                 no_alias = Alias.blacklist
                 for pdir in path:
                     try:
                         os.chdir(pdir)
                     except OSError:
                         continue
 
                     # for python 3.6+ rewrite to: with os.scandir(pdir) as dirlist:
                     dirlist = os.scandir(pdir)
                     for ff in dirlist:
                         fname = ff.name
                         base, ext = os.path.splitext(fname)
                         if self.isexec(ff) and base.lower() not in no_alias:
                             if ext.lower() == '.exe':
                                 fname = base
                                 try:
                                     # Removes dots from the name since ipython
                                     # will assume names with dots to be python.
                                     self.shell.alias_manager.define_alias(
                                         base.lower().replace('.',''), fname)
                                 except InvalidAliasError:
                                     pass
                                 syscmdlist.append(fname)
 
             self.shell.db['syscmdlist'] = syscmdlist
         finally:
             os.chdir(savedir)
 
     @skip_doctest
     @line_magic
     def pwd(self, parameter_s=''):
         """Return the current working directory path.
 
         Examples
         --------
         ::
 
           In [9]: pwd
           Out[9]: '/home/tsuser/sprint/ipython'
         """
         try:
             return os.getcwd()
         except FileNotFoundError as e:
             raise UsageError("CWD no longer exists - please use %cd to change directory.") from e
 
     @skip_doctest
     @line_magic
     def cd(self, parameter_s=''):
         """Change the current working directory.
 
         This command automatically maintains an internal list of directories
         you visit during your IPython session, in the variable ``_dh``. The
         command :magic:`%dhist` shows this history nicely formatted. You can
         also do ``cd -<tab>`` to see directory history conveniently.
         Usage:
 
           - ``cd 'dir'``: changes to directory 'dir'.
           - ``cd -``: changes to the last visited directory.
           - ``cd -<n>``: changes to the n-th directory in the directory history.
           - ``cd --foo``: change to directory that matches 'foo' in history
           - ``cd -b <bookmark_name>``: jump to a bookmark set by %bookmark
           - Hitting a tab key after ``cd -b`` allows you to tab-complete
             bookmark names.
 
           .. note::
             ``cd <bookmark_name>`` is enough if there is no directory
             ``<bookmark_name>``, but a bookmark with the name exists.
 
 
         Options:
 
         -q               Be quiet. Do not print the working directory after the
                          cd command is executed. By default IPython's cd
                          command does print this directory, since the default
                          prompts do not display path information.
 
         .. note::
            Note that ``!cd`` doesn't work for this purpose because the shell
            where ``!command`` runs is immediately discarded after executing
            'command'.
 
 
         Examples
         --------
         ::
 
           In [10]: cd parent/child
           /home/tsuser/parent/child
         """
 
         try:
             oldcwd = os.getcwd()
         except FileNotFoundError:
             # Happens if the CWD has been deleted.
             oldcwd = None
 
         numcd = re.match(r'(-)(\d+)$',parameter_s)
         # jump in directory history by number
         if numcd:
             nn = int(numcd.group(2))
             try:
                 ps = self.shell.user_ns['_dh'][nn]
             except IndexError:
                 print('The requested directory does not exist in history.')
                 return
             else:
                 opts = {}
         elif parameter_s.startswith('--'):
             ps = None
             fallback = None
             pat = parameter_s[2:]
             dh = self.shell.user_ns['_dh']
             # first search only by basename (last component)
             for ent in reversed(dh):
                 if pat in os.path.basename(ent) and os.path.isdir(ent):
                     ps = ent
                     break
 
                 if fallback is None and pat in ent and os.path.isdir(ent):
                     fallback = ent
 
             # if we have no last part match, pick the first full path match
             if ps is None:
                 ps = fallback
 
             if ps is None:
                 print("No matching entry in directory history")
                 return
             else:
                 opts = {}
 
 
         else:
             opts, ps = self.parse_options(parameter_s, 'qb', mode='string')
         # jump to previous
         if ps == '-':
             try:
                 ps = self.shell.user_ns['_dh'][-2]
             except IndexError as e:
                 raise UsageError('%cd -: No previous directory to change to.') from e
         # jump to bookmark if needed
         else:
             if not os.path.isdir(ps) or 'b' in opts:
                 bkms = self.shell.db.get('bookmarks', {})
 
                 if ps in bkms:
                     target = bkms[ps]
                     print('(bookmark:%s) -> %s' % (ps, target))
                     ps = target
                 else:
                     if 'b' in opts:
                         raise UsageError("Bookmark '%s' not found.  "
                               "Use '%%bookmark -l' to see your bookmarks." % ps)
 
         # at this point ps should point to the target dir
         if ps:
             try:
                 os.chdir(os.path.expanduser(ps))
                 if hasattr(self.shell, 'term_title') and self.shell.term_title:
                     set_term_title(self.shell.term_title_format.format(cwd=abbrev_cwd()))
             except OSError:
                 print(sys.exc_info()[1])
             else:
                 cwd = os.getcwd()
                 dhist = self.shell.user_ns['_dh']
                 if oldcwd != cwd:
                     dhist.append(cwd)
                     self.shell.db['dhist'] = compress_dhist(dhist)[-100:]
 
         else:
             os.chdir(self.shell.home_dir)
             if hasattr(self.shell, 'term_title') and self.shell.term_title:
                 set_term_title(self.shell.term_title_format.format(cwd="~"))
             cwd = os.getcwd()
             dhist = self.shell.user_ns['_dh']
 
             if oldcwd != cwd:
                 dhist.append(cwd)
                 self.shell.db['dhist'] = compress_dhist(dhist)[-100:]
         if not 'q' in opts and not self.cd_force_quiet and self.shell.user_ns['_dh']:
             print(self.shell.user_ns['_dh'][-1])
 
     @line_magic
     def env(self, parameter_s=''):
         """Get, set, or list environment variables.
 
         Usage:\\
 
           :``%env``: lists all environment variables/values
           :``%env var``: get value for var
           :``%env var val``: set value for var
           :``%env var=val``: set value for var
           :``%env var=$val``: set value for var, using python expansion if possible
         """
         if parameter_s.strip():
             split = '=' if '=' in parameter_s else ' '
             bits = parameter_s.split(split)
             if len(bits) == 1:
                 key = parameter_s.strip()
                 if key in os.environ:
                     return os.environ[key]
                 else:
                     err = "Environment does not have key: {0}".format(key)
                     raise UsageError(err)
             if len(bits) > 1:
                 return self.set_env(parameter_s)
         env = dict(os.environ)
         # hide likely secrets when printing the whole environment
         for key in list(env):
             if any(s in key.lower() for s in ('key', 'token', 'secret')):
                 env[key] = '<hidden>'
 
         return env
 
     @line_magic
     def set_env(self, parameter_s):
         """Set environment variables.  Assumptions are that either "val" is a
         name in the user namespace, or val is something that evaluates to a
         string.
 
         Usage:\\
           %set_env var val: set value for var
           %set_env var=val: set value for var
           %set_env var=$val: set value for var, using python expansion if possible
         """
         split = '=' if '=' in parameter_s else ' '
         bits = parameter_s.split(split, 1)
         if not parameter_s.strip() or len(bits)<2:
             raise UsageError("usage is 'set_env var=val'")
         var = bits[0].strip()
         val = bits[1].strip()
         if re.match(r'.*\s.*', var):
             # an environment variable with whitespace is almost certainly
             # not what the user intended.  what's more likely is the wrong
             # split was chosen, ie for "set_env cmd_args A=B", we chose
             # '=' for the split and should have chosen ' '.  to get around
             # this, users should just assign directly to os.environ or use
             # standard magic {var} expansion.
             err = "refusing to set env var with whitespace: '{0}'"
             err = err.format(val)
             raise UsageError(err)
         os.environ[var] = val
         print('env: {0}={1}'.format(var,val))
 
     @line_magic
     def pushd(self, parameter_s=''):
         """Place the current dir on stack and change directory.
 
         Usage:\\
           %pushd ['dirname']
         """
 
         dir_s = self.shell.dir_stack
         tgt = os.path.expanduser(parameter_s)
         cwd = os.getcwd().replace(self.shell.home_dir,'~')
         if tgt:
             self.cd(parameter_s)
         dir_s.insert(0,cwd)
         return self.shell.run_line_magic('dirs', '')
 
     @line_magic
     def popd(self, parameter_s=''):
         """Change to directory popped off the top of the stack.
         """
         if not self.shell.dir_stack:
             raise UsageError("%popd on empty stack")
         top = self.shell.dir_stack.pop(0)
         self.cd(top)
         print("popd ->",top)
 
     @line_magic
     def dirs(self, parameter_s=''):
         """Return the current directory stack."""
 
         return self.shell.dir_stack
 
     @line_magic
     def dhist(self, parameter_s=''):
         """Print your history of visited directories.
 
         %dhist       -> print full history\\
         %dhist n     -> print last n entries only\\
         %dhist n1 n2 -> print entries between n1 and n2 (n2 not included)\\
 
         This history is automatically maintained by the %cd command, and
         always available as the global list variable _dh. You can use %cd -<n>
         to go to directory number <n>.
 
         Note that most of time, you should view directory history by entering
         cd -<TAB>.
 
         """
 
         dh = self.shell.user_ns['_dh']
         if parameter_s:
             try:
                 args = map(int,parameter_s.split())
             except:
                 self.arg_err(self.dhist)
                 return
             if len(args) == 1:
                 ini,fin = max(len(dh)-(args[0]),0),len(dh)
             elif len(args) == 2:
                 ini,fin = args
                 fin = min(fin, len(dh))
             else:
                 self.arg_err(self.dhist)
                 return
         else:
             ini,fin = 0,len(dh)
         print('Directory history (kept in _dh)')
         for i in range(ini, fin):
             print("%d: %s" % (i, dh[i]))
 
     @skip_doctest
     @line_magic
     def sc(self, parameter_s=''):
         """Shell capture - run shell command and capture output (DEPRECATED use !).
 
         DEPRECATED. Suboptimal, retained for backwards compatibility.
 
         You should use the form 'var = !command' instead. Example:
 
          "%sc -l myfiles = ls ~" should now be written as
 
          "myfiles = !ls ~"
 
         myfiles.s, myfiles.l and myfiles.n still apply as documented
         below.
 
         --
         %sc [options] varname=command
 
         IPython will run the given command using commands.getoutput(), and
         will then update the user's interactive namespace with a variable
         called varname, containing the value of the call.  Your command can
         contain shell wildcards, pipes, etc.
 
         The '=' sign in the syntax is mandatory, and the variable name you
         supply must follow Python's standard conventions for valid names.
 
         (A special format without variable name exists for internal use)
 
         Options:
 
           -l: list output.  Split the output on newlines into a list before
           assigning it to the given variable.  By default the output is stored
           as a single string.
 
           -v: verbose.  Print the contents of the variable.
 
         In most cases you should not need to split as a list, because the
         returned value is a special type of string which can automatically
         provide its contents either as a list (split on newlines) or as a
         space-separated string.  These are convenient, respectively, either
         for sequential processing or to be passed to a shell command.
 
         For example::
 
             # Capture into variable a
             In [1]: sc a=ls *py
 
             # a is a string with embedded newlines
             In [2]: a
             Out[2]: 'setup.py\\nwin32_manual_post_install.py'
 
             # which can be seen as a list:
             In [3]: a.l
             Out[3]: ['setup.py', 'win32_manual_post_install.py']
 
             # or as a whitespace-separated string:
             In [4]: a.s
             Out[4]: 'setup.py win32_manual_post_install.py'
 
             # a.s is useful to pass as a single command line:
             In [5]: !wc -l $a.s
               146 setup.py
               130 win32_manual_post_install.py
               276 total
 
             # while the list form is useful to loop over:
             In [6]: for f in a.l:
               ...:      !wc -l $f
               ...:
             146 setup.py
             130 win32_manual_post_install.py
 
         Similarly, the lists returned by the -l option are also special, in
         the sense that you can equally invoke the .s attribute on them to
         automatically get a whitespace-separated string from their contents::
 
             In [7]: sc -l b=ls *py
 
             In [8]: b
             Out[8]: ['setup.py', 'win32_manual_post_install.py']
 
             In [9]: b.s
             Out[9]: 'setup.py win32_manual_post_install.py'
 
         In summary, both the lists and strings used for output capture have
         the following special attributes::
 
             .l (or .list) : value as list.
             .n (or .nlstr): value as newline-separated string.
             .s (or .spstr): value as space-separated string.
         """
 
         opts,args = self.parse_options(parameter_s, 'lv')
         # Try to get a variable name and command to run
         try:
             # the variable name must be obtained from the parse_options
             # output, which uses shlex.split to strip options out.
             var,_ = args.split('=', 1)
             var = var.strip()
             # But the command has to be extracted from the original input
             # parameter_s, not on what parse_options returns, to avoid the
             # quote stripping which shlex.split performs on it.
             _,cmd = parameter_s.split('=', 1)
         except ValueError:
             var,cmd = '',''
         # If all looks ok, proceed
         split = 'l' in opts
         out = self.shell.getoutput(cmd, split=split)
         if 'v' in opts:
             print('%s ==\n%s' % (var, pformat(out)))
         if var:
             self.shell.user_ns.update({var:out})
         else:
             return out
 
     @line_cell_magic
     def sx(self, line='', cell=None):
         """Shell execute - run shell command and capture output (!! is short-hand).
 
         %sx command
 
         IPython will run the given command using commands.getoutput(), and
         return the result formatted as a list (split on '\\n').  Since the
         output is _returned_, it will be stored in ipython's regular output
         cache Out[N] and in the '_N' automatic variables.
 
         Notes:
 
         1) If an input line begins with '!!', then %sx is automatically
         invoked.  That is, while::
 
           !ls
 
         causes ipython to simply issue system('ls'), typing::
 
           !!ls
 
         is a shorthand equivalent to::
 
           %sx ls
 
         2) %sx differs from %sc in that %sx automatically splits into a list,
         like '%sc -l'.  The reason for this is to make it as easy as possible
         to process line-oriented shell output via further python commands.
         %sc is meant to provide much finer control, but requires more
         typing.
 
         3) Just like %sc -l, this is a list with special attributes:
         ::
 
           .l (or .list) : value as list.
           .n (or .nlstr): value as newline-separated string.
           .s (or .spstr): value as whitespace-separated string.
 
         This is very useful when trying to use such lists as arguments to
         system commands."""
         
         if cell is None:
             # line magic
             return self.shell.getoutput(line)
         else:
             opts,args = self.parse_options(line, '', 'out=')
             output = self.shell.getoutput(cell)
             out_name = opts.get('out', opts.get('o'))
             if out_name:
                 self.shell.user_ns[out_name] = output
             else:
                 return output
 
     system = line_cell_magic('system')(sx)
     bang = cell_magic('!')(sx)
 
     @line_magic
     def bookmark(self, parameter_s=''):
         """Manage IPython's bookmark system.
 
         %bookmark <name>       - set bookmark to current dir
         %bookmark <name> <dir> - set bookmark to <dir>
         %bookmark -l           - list all bookmarks
         %bookmark -d <name>    - remove bookmark
         %bookmark -r           - remove all bookmarks
 
         You can later on access a bookmarked folder with::
 
           %cd -b <name>
 
         or simply '%cd <name>' if there is no directory called <name> AND
         there is such a bookmark defined.
 
         Your bookmarks persist through IPython sessions, but they are
         associated with each profile."""
 
         opts,args = self.parse_options(parameter_s,'drl',mode='list')
         if len(args) > 2:
             raise UsageError("%bookmark: too many arguments")
 
         bkms = self.shell.db.get('bookmarks',{})
 
         if 'd' in opts:
             try:
                 todel = args[0]
             except IndexError as e:
                 raise UsageError(
                     "%bookmark -d: must provide a bookmark to delete") from e
             else:
                 try:
                     del bkms[todel]
                 except KeyError as e:
                     raise UsageError(
                         "%%bookmark -d: Can't delete bookmark '%s'" % todel) from e
 
         elif 'r' in opts:
             bkms = {}
         elif 'l' in opts:
             bks = sorted(bkms)
             if bks:
                 size = max(map(len, bks))
             else:
                 size = 0
             fmt = '%-'+str(size)+'s -> %s'
             print('Current bookmarks:')
             for bk in bks:
                 print(fmt % (bk, bkms[bk]))
         else:
             if not args:
                 raise UsageError("%bookmark: You must specify the bookmark name")
             elif len(args)==1:
                 bkms[args[0]] = os.getcwd()
             elif len(args)==2:
                 bkms[args[0]] = args[1]
         self.shell.db['bookmarks'] = bkms
 
     @line_magic
     def pycat(self, parameter_s=''):
         """Show a syntax-highlighted file through a pager.
 
         This magic is similar to the cat utility, but it will assume the file
         to be Python source and will show it with syntax highlighting.
 
         This magic command can either take a local filename, an url,
         an history range (see %history) or a macro as argument.
 
         If no parameter is given, prints out history of current session up to
         this point. ::
 
         %pycat myscript.py
         %pycat 7-27
         %pycat myMacro
         %pycat http://www.example.com/myscript.py
         """
         try:
             cont = self.shell.find_user_code(parameter_s, skip_encoding_cookie=False)
         except (ValueError, IOError):
             print("Error: no such file, variable, URL, history range or macro")
             return
 
         page.page(self.shell.pycolorize(source_to_unicode(cont)))
 
     @magic_arguments.magic_arguments()
     @magic_arguments.argument(
         '-a', '--append', action='store_true', default=False,
         help='Append contents of the cell to an existing file. '
              'The file will be created if it does not exist.'
     )
     @magic_arguments.argument(
         'filename', type=str,
         help='file to write'
     )
     @cell_magic
     def writefile(self, line, cell):
         """Write the contents of the cell to a file.
         
         The file will be overwritten unless the -a (--append) flag is specified.
         """
         args = magic_arguments.parse_argstring(self.writefile, line)
         if re.match(r'^(\'.*\')|(".*")$', args.filename):
             filename = os.path.expanduser(args.filename[1:-1])
         else:
             filename = os.path.expanduser(args.filename)
             
         if os.path.exists(filename):
             if args.append:
                 print("Appending to %s" % filename)
             else:
                 print("Overwriting %s" % filename)
         else:
             print("Writing %s" % filename)
         
         mode = 'a' if args.append else 'w'
         with io.open(filename, mode, encoding='utf-8') as f:
             f.write(cell)