Links cleanup, in particular peps have moved

IPython/core/magics/osm.py

@@ -1,854 +1,854 b''
 """Implementation of magic functions for interaction with the OS.
 
 Note: this module is named 'osm' instead of 'os' to avoid a collision with the
 builtin.
 """
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 import io
 import os
 import re
 import sys
 from pprint import pformat
 
 from IPython.core import magic_arguments
 from IPython.core import oinspect
 from IPython.core import page
 from IPython.core.alias import AliasError, Alias
 from IPython.core.error import UsageError
 from IPython.core.magic import  (
     Magics, compress_dhist, magics_class, line_magic, cell_magic, line_cell_magic
 )
 from IPython.testing.skipdoctest import skip_doctest
 from IPython.utils.openpy import source_to_unicode
 from IPython.utils.process import abbrev_cwd
 from IPython.utils.terminal import set_term_title
 from traitlets import Bool
 from warnings import warn
 
 
 @magics_class
 class OSMagics(Magics):
     """Magics to interact with the underlying OS (shell-type functionality).
     """
 
     cd_force_quiet = Bool(False,
         help="Force %cd magic to be quiet even if -q is not passed."
     ).tag(config=True)
 
     def __init__(self, shell=None, **kwargs):
 
         # Now define isexec in a cross platform manner.
         self.is_posix = False
         self.execre = None
         if os.name == 'posix':
             self.is_posix = True
         else:
             try:
                 winext = os.environ['pathext'].replace(';','|').replace('.','')
             except KeyError:
                 winext = 'exe|com|bat|py'
             try:
                 self.execre = re.compile(r'(.*)\.(%s)$' % winext,re.IGNORECASE)
             except re.error:
                 warn("Seems like your pathext environmental "
                      "variable is malformed. Please check it to "
                      "enable a proper handle of file extensions "
                      "managed for your system")
                 winext = 'exe|com|bat|py'
                 self.execre = re.compile(r'(.*)\.(%s)$' % winext,re.IGNORECASE)
 
         # call up the chain
         super().__init__(shell=shell, **kwargs)
 
 
     def _isexec_POSIX(self, file):
         """
         Test for executable on a POSIX system
         """
         if os.access(file.path, os.X_OK):
             # will fail on maxOS if access is not X_OK
             return file.is_file()
         return False
 
 
     
     def _isexec_WIN(self, file):
         """
         Test for executable file on non POSIX system
         """
         return file.is_file() and self.execre.match(file.name) is not None
 
     def isexec(self, file):
         """
         Test for executable file on non POSIX system
         """
         if self.is_posix:
             return self._isexec_POSIX(file)
         else:
             return self._isexec_WIN(file)
 
 
     @skip_doctest
     @line_magic
     def alias(self, parameter_s=''):
         """Define an alias for a system command.
 
         '%alias alias_name cmd' defines 'alias_name' as an alias for 'cmd'
 
         Then, typing 'alias_name params' will execute the system command 'cmd
         params' (from your underlying operating system).
 
         Aliases have lower precedence than magic functions and Python normal
         variables, so if 'foo' is both a Python variable and an alias, the
         alias can not be executed until 'del foo' removes the Python variable.
 
         You can use the %l specifier in an alias definition to represent the
         whole line when the alias is called.  For example::
 
           In [2]: alias bracket echo "Input in brackets: <%l>"
           In [3]: bracket hello world
           Input in brackets: <hello world>
 
         You can also define aliases with parameters using %s specifiers (one
         per parameter)::
 
           In [1]: alias parts echo first %s second %s
           In [2]: %parts A B
           first A second B
           In [3]: %parts A
           Incorrect number of arguments: 2 expected.
           parts is an alias to: 'echo first %s second %s'
 
         Note that %l and %s are mutually exclusive.  You can only use one or
         the other in your aliases.
 
         Aliases expand Python variables just like system calls using ! or !!
         do: all expressions prefixed with '$' get expanded.  For details of
         the semantic rules, see PEP-215:
-        https://www.python.org/dev/peps/pep-0215/.  This is the library used by
+        https://peps.python.org/pep-0215/.  This is the library used by
         IPython for variable expansion.  If you want to access a true shell
         variable, an extra $ is necessary to prevent its expansion by
         IPython::
 
           In [6]: alias show echo
           In [7]: PATH='A Python string'
           In [8]: show $PATH
           A Python string
           In [9]: show $$PATH
           /usr/local/lf9560/bin:/usr/local/intel/compiler70/ia32/bin:...
 
         You can use the alias facility to access all of $PATH.  See the %rehashx
         function, which automatically creates aliases for the contents of your
         $PATH.
 
         If called with no parameters, %alias prints the current alias table
         for your system.  For posix systems, the default aliases are 'cat',
         'cp', 'mv', 'rm', 'rmdir', and 'mkdir', and other platform-specific
         aliases are added.  For windows-based systems, the default aliases are
         'copy', 'ddir', 'echo', 'ls', 'ldir', 'mkdir', 'ren', and 'rmdir'.
 
         You can see the definition of alias by adding a question mark in the
         end::
 
           In [1]: cat?
           Repr: <alias cat for 'cat'>"""
 
         par = parameter_s.strip()
         if not par:
             aliases = sorted(self.shell.alias_manager.aliases)
             # stored = self.shell.db.get('stored_aliases', {} )
             # for k, v in stored:
             #     atab.append(k, v[0])
 
             print("Total number of aliases:", len(aliases))
             sys.stdout.flush()
             return aliases
 
         # Now try to define a new one
         try:
             alias,cmd = par.split(None, 1)
         except TypeError:
             print(oinspect.getdoc(self.alias))
             return
         
         try:
             self.shell.alias_manager.define_alias(alias, cmd)
         except AliasError as e:
             print(e)
     # end magic_alias
 
     @line_magic
     def unalias(self, parameter_s=''):
         """Remove an alias"""
 
         aname = parameter_s.strip()
         try:
             self.shell.alias_manager.undefine_alias(aname)
         except ValueError as e:
             print(e)
             return
         
         stored = self.shell.db.get('stored_aliases', {} )
         if aname in stored:
             print("Removing %stored alias",aname)
             del stored[aname]
             self.shell.db['stored_aliases'] = stored
 
     @line_magic
     def rehashx(self, parameter_s=''):
         """Update the alias table with all executable files in $PATH.
 
         rehashx explicitly checks that every entry in $PATH is a file
         with execute access (os.X_OK).
 
         Under Windows, it checks executability as a match against a
         '|'-separated string of extensions, stored in the IPython config
         variable win_exec_ext.  This defaults to 'exe|com|bat'.
 
         This function also resets the root module cache of module completer,
         used on slow filesystems.
         """
         from IPython.core.alias import InvalidAliasError
 
         # for the benefit of module completer in ipy_completers.py
         del self.shell.db['rootmodules_cache']
 
         path = [os.path.abspath(os.path.expanduser(p)) for p in
             os.environ.get('PATH','').split(os.pathsep)]
 
         syscmdlist = []
         savedir = os.getcwd()
 
         # Now walk the paths looking for executables to alias.
         try:
             # write the whole loop for posix/Windows so we don't have an if in
             # the innermost part
             if self.is_posix:
                 for pdir in path:
                     try:
                         os.chdir(pdir)
                     except OSError:
                         continue
 
                     # for python 3.6+ rewrite to: with os.scandir(pdir) as dirlist:
                     dirlist = os.scandir(path=pdir)
                     for ff in dirlist:
                         if self.isexec(ff):
                             fname = ff.name
                             try:
                                 # Removes dots from the name since ipython
                                 # will assume names with dots to be python.
                                 if not self.shell.alias_manager.is_alias(fname):
                                     self.shell.alias_manager.define_alias(
                                         fname.replace('.',''), fname)
                             except InvalidAliasError:
                                 pass
                             else:
                                 syscmdlist.append(fname)
             else:
                 no_alias = Alias.blacklist
                 for pdir in path:
                     try:
                         os.chdir(pdir)
                     except OSError:
                         continue
 
                     # for python 3.6+ rewrite to: with os.scandir(pdir) as dirlist:
                     dirlist = os.scandir(pdir)
                     for ff in dirlist:
                         fname = ff.name
                         base, ext = os.path.splitext(fname)
                         if self.isexec(ff) and base.lower() not in no_alias:
                             if ext.lower() == '.exe':
                                 fname = base
                                 try:
                                     # Removes dots from the name since ipython
                                     # will assume names with dots to be python.
                                     self.shell.alias_manager.define_alias(
                                         base.lower().replace('.',''), fname)
                                 except InvalidAliasError:
                                     pass
                                 syscmdlist.append(fname)
 
             self.shell.db['syscmdlist'] = syscmdlist
         finally:
             os.chdir(savedir)
 
     @skip_doctest
     @line_magic
     def pwd(self, parameter_s=''):
         """Return the current working directory path.
 
         Examples
         --------
         ::
 
           In [9]: pwd
           Out[9]: '/home/tsuser/sprint/ipython'
         """
         try:
             return os.getcwd()
         except FileNotFoundError as e:
             raise UsageError("CWD no longer exists - please use %cd to change directory.") from e
 
     @skip_doctest
     @line_magic
     def cd(self, parameter_s=''):
         """Change the current working directory.
 
         This command automatically maintains an internal list of directories
         you visit during your IPython session, in the variable ``_dh``. The
         command :magic:`%dhist` shows this history nicely formatted. You can
         also do ``cd -<tab>`` to see directory history conveniently.
         Usage:
 
           - ``cd 'dir'``: changes to directory 'dir'.
           - ``cd -``: changes to the last visited directory.
           - ``cd -<n>``: changes to the n-th directory in the directory history.
           - ``cd --foo``: change to directory that matches 'foo' in history
           - ``cd -b <bookmark_name>``: jump to a bookmark set by %bookmark
           - Hitting a tab key after ``cd -b`` allows you to tab-complete
             bookmark names.
 
           .. note::
             ``cd <bookmark_name>`` is enough if there is no directory
             ``<bookmark_name>``, but a bookmark with the name exists.
 
         Options:
 
         -q               Be quiet. Do not print the working directory after the
                          cd command is executed. By default IPython's cd
                          command does print this directory, since the default
                          prompts do not display path information.
 
         .. note::
            Note that ``!cd`` doesn't work for this purpose because the shell
            where ``!command`` runs is immediately discarded after executing
            'command'.
 
         Examples
         --------
         ::
 
           In [10]: cd parent/child
           /home/tsuser/parent/child
         """
 
         try:
             oldcwd = os.getcwd()
         except FileNotFoundError:
             # Happens if the CWD has been deleted.
             oldcwd = None
 
         numcd = re.match(r'(-)(\d+)$',parameter_s)
         # jump in directory history by number
         if numcd:
             nn = int(numcd.group(2))
             try:
                 ps = self.shell.user_ns['_dh'][nn]
             except IndexError:
                 print('The requested directory does not exist in history.')
                 return
             else:
                 opts = {}
         elif parameter_s.startswith('--'):
             ps = None
             fallback = None
             pat = parameter_s[2:]
             dh = self.shell.user_ns['_dh']
             # first search only by basename (last component)
             for ent in reversed(dh):
                 if pat in os.path.basename(ent) and os.path.isdir(ent):
                     ps = ent
                     break
 
                 if fallback is None and pat in ent and os.path.isdir(ent):
                     fallback = ent
 
             # if we have no last part match, pick the first full path match
             if ps is None:
                 ps = fallback
 
             if ps is None:
                 print("No matching entry in directory history")
                 return
             else:
                 opts = {}
 
 
         else:
             opts, ps = self.parse_options(parameter_s, 'qb', mode='string')
         # jump to previous
         if ps == '-':
             try:
                 ps = self.shell.user_ns['_dh'][-2]
             except IndexError as e:
                 raise UsageError('%cd -: No previous directory to change to.') from e
         # jump to bookmark if needed
         else:
             if not os.path.isdir(ps) or 'b' in opts:
                 bkms = self.shell.db.get('bookmarks', {})
 
                 if ps in bkms:
                     target = bkms[ps]
                     print('(bookmark:%s) -> %s' % (ps, target))
                     ps = target
                 else:
                     if 'b' in opts:
                         raise UsageError("Bookmark '%s' not found.  "
                               "Use '%%bookmark -l' to see your bookmarks." % ps)
 
         # at this point ps should point to the target dir
         if ps:
             try:
                 os.chdir(os.path.expanduser(ps))
                 if hasattr(self.shell, 'term_title') and self.shell.term_title:
                     set_term_title(self.shell.term_title_format.format(cwd=abbrev_cwd()))
             except OSError:
                 print(sys.exc_info()[1])
             else:
                 cwd = os.getcwd()
                 dhist = self.shell.user_ns['_dh']
                 if oldcwd != cwd:
                     dhist.append(cwd)
                     self.shell.db['dhist'] = compress_dhist(dhist)[-100:]
 
         else:
             os.chdir(self.shell.home_dir)
             if hasattr(self.shell, 'term_title') and self.shell.term_title:
                 set_term_title(self.shell.term_title_format.format(cwd="~"))
             cwd = os.getcwd()
             dhist = self.shell.user_ns['_dh']
 
             if oldcwd != cwd:
                 dhist.append(cwd)
                 self.shell.db['dhist'] = compress_dhist(dhist)[-100:]
         if not 'q' in opts and not self.cd_force_quiet and self.shell.user_ns['_dh']:
             print(self.shell.user_ns['_dh'][-1])
 
     @line_magic
     def env(self, parameter_s=''):
         """Get, set, or list environment variables.
 
         Usage:\\
 
           :``%env``: lists all environment variables/values
           :``%env var``: get value for var
           :``%env var val``: set value for var
           :``%env var=val``: set value for var
           :``%env var=$val``: set value for var, using python expansion if possible
         """
         if parameter_s.strip():
             split = '=' if '=' in parameter_s else ' '
             bits = parameter_s.split(split)
             if len(bits) == 1:
                 key = parameter_s.strip()
                 if key in os.environ:
                     return os.environ[key]
                 else:
                     err = "Environment does not have key: {0}".format(key)
                     raise UsageError(err)
             if len(bits) > 1:
                 return self.set_env(parameter_s)
         env = dict(os.environ)
         # hide likely secrets when printing the whole environment
         for key in list(env):
             if any(s in key.lower() for s in ('key', 'token', 'secret')):
                 env[key] = '<hidden>'
 
         return env
 
     @line_magic
     def set_env(self, parameter_s):
         """Set environment variables.  Assumptions are that either "val" is a
         name in the user namespace, or val is something that evaluates to a
         string.
 
         Usage:\\
           %set_env var val: set value for var
           %set_env var=val: set value for var
           %set_env var=$val: set value for var, using python expansion if possible
         """
         split = '=' if '=' in parameter_s else ' '
         bits = parameter_s.split(split, 1)
         if not parameter_s.strip() or len(bits)<2:
             raise UsageError("usage is 'set_env var=val'")
         var = bits[0].strip()
         val = bits[1].strip()
         if re.match(r'.*\s.*', var):
             # an environment variable with whitespace is almost certainly
             # not what the user intended.  what's more likely is the wrong
             # split was chosen, ie for "set_env cmd_args A=B", we chose
             # '=' for the split and should have chosen ' '.  to get around
             # this, users should just assign directly to os.environ or use
             # standard magic {var} expansion.
             err = "refusing to set env var with whitespace: '{0}'"
             err = err.format(val)
             raise UsageError(err)
         os.environ[var] = val
         print('env: {0}={1}'.format(var,val))
 
     @line_magic
     def pushd(self, parameter_s=''):
         """Place the current dir on stack and change directory.
 
         Usage:\\
           %pushd ['dirname']
         """
 
         dir_s = self.shell.dir_stack
         tgt = os.path.expanduser(parameter_s)
         cwd = os.getcwd().replace(self.shell.home_dir,'~')
         if tgt:
             self.cd(parameter_s)
         dir_s.insert(0,cwd)
         return self.shell.run_line_magic('dirs', '')
 
     @line_magic
     def popd(self, parameter_s=''):
         """Change to directory popped off the top of the stack.
         """
         if not self.shell.dir_stack:
             raise UsageError("%popd on empty stack")
         top = self.shell.dir_stack.pop(0)
         self.cd(top)
         print("popd ->",top)
 
     @line_magic
     def dirs(self, parameter_s=''):
         """Return the current directory stack."""
 
         return self.shell.dir_stack
 
     @line_magic
     def dhist(self, parameter_s=''):
         """Print your history of visited directories.
 
         %dhist       -> print full history\\
         %dhist n     -> print last n entries only\\
         %dhist n1 n2 -> print entries between n1 and n2 (n2 not included)\\
 
         This history is automatically maintained by the %cd command, and
         always available as the global list variable _dh. You can use %cd -<n>
         to go to directory number <n>.
 
         Note that most of time, you should view directory history by entering
         cd -<TAB>.
 
         """
 
         dh = self.shell.user_ns['_dh']
         if parameter_s:
             try:
                 args = map(int,parameter_s.split())
             except:
                 self.arg_err(self.dhist)
                 return
             if len(args) == 1:
                 ini,fin = max(len(dh)-(args[0]),0),len(dh)
             elif len(args) == 2:
                 ini,fin = args
                 fin = min(fin, len(dh))
             else:
                 self.arg_err(self.dhist)
                 return
         else:
             ini,fin = 0,len(dh)
         print('Directory history (kept in _dh)')
         for i in range(ini, fin):
             print("%d: %s" % (i, dh[i]))
 
     @skip_doctest
     @line_magic
     def sc(self, parameter_s=''):
         """Shell capture - run shell command and capture output (DEPRECATED use !).
 
         DEPRECATED. Suboptimal, retained for backwards compatibility.
 
         You should use the form 'var = !command' instead. Example:
 
          "%sc -l myfiles = ls ~" should now be written as
 
          "myfiles = !ls ~"
 
         myfiles.s, myfiles.l and myfiles.n still apply as documented
         below.
 
         --
         %sc [options] varname=command
 
         IPython will run the given command using commands.getoutput(), and
         will then update the user's interactive namespace with a variable
         called varname, containing the value of the call.  Your command can
         contain shell wildcards, pipes, etc.
 
         The '=' sign in the syntax is mandatory, and the variable name you
         supply must follow Python's standard conventions for valid names.
 
         (A special format without variable name exists for internal use)
 
         Options:
 
           -l: list output.  Split the output on newlines into a list before
           assigning it to the given variable.  By default the output is stored
           as a single string.
 
           -v: verbose.  Print the contents of the variable.
 
         In most cases you should not need to split as a list, because the
         returned value is a special type of string which can automatically
         provide its contents either as a list (split on newlines) or as a
         space-separated string.  These are convenient, respectively, either
         for sequential processing or to be passed to a shell command.
 
         For example::
 
             # Capture into variable a
             In [1]: sc a=ls *py
 
             # a is a string with embedded newlines
             In [2]: a
             Out[2]: 'setup.py\\nwin32_manual_post_install.py'
 
             # which can be seen as a list:
             In [3]: a.l
             Out[3]: ['setup.py', 'win32_manual_post_install.py']
 
             # or as a whitespace-separated string:
             In [4]: a.s
             Out[4]: 'setup.py win32_manual_post_install.py'
 
             # a.s is useful to pass as a single command line:
             In [5]: !wc -l $a.s
               146 setup.py
               130 win32_manual_post_install.py
               276 total
 
             # while the list form is useful to loop over:
             In [6]: for f in a.l:
                ...:      !wc -l $f
                ...:
             146 setup.py
             130 win32_manual_post_install.py
 
         Similarly, the lists returned by the -l option are also special, in
         the sense that you can equally invoke the .s attribute on them to
         automatically get a whitespace-separated string from their contents::
 
             In [7]: sc -l b=ls *py
 
             In [8]: b
             Out[8]: ['setup.py', 'win32_manual_post_install.py']
 
             In [9]: b.s
             Out[9]: 'setup.py win32_manual_post_install.py'
 
         In summary, both the lists and strings used for output capture have
         the following special attributes::
 
             .l (or .list) : value as list.
             .n (or .nlstr): value as newline-separated string.
             .s (or .spstr): value as space-separated string.
         """
 
         opts,args = self.parse_options(parameter_s, 'lv')
         # Try to get a variable name and command to run
         try:
             # the variable name must be obtained from the parse_options
             # output, which uses shlex.split to strip options out.
             var,_ = args.split('=', 1)
             var = var.strip()
             # But the command has to be extracted from the original input
             # parameter_s, not on what parse_options returns, to avoid the
             # quote stripping which shlex.split performs on it.
             _,cmd = parameter_s.split('=', 1)
         except ValueError:
             var,cmd = '',''
         # If all looks ok, proceed
         split = 'l' in opts
         out = self.shell.getoutput(cmd, split=split)
         if 'v' in opts:
             print('%s ==\n%s' % (var, pformat(out)))
         if var:
             self.shell.user_ns.update({var:out})
         else:
             return out
 
     @line_cell_magic
     def sx(self, line='', cell=None):
         """Shell execute - run shell command and capture output (!! is short-hand).
 
         %sx command
 
         IPython will run the given command using commands.getoutput(), and
         return the result formatted as a list (split on '\\n').  Since the
         output is _returned_, it will be stored in ipython's regular output
         cache Out[N] and in the '_N' automatic variables.
 
         Notes:
 
         1) If an input line begins with '!!', then %sx is automatically
         invoked.  That is, while::
 
           !ls
 
         causes ipython to simply issue system('ls'), typing::
 
           !!ls
 
         is a shorthand equivalent to::
 
           %sx ls
 
         2) %sx differs from %sc in that %sx automatically splits into a list,
         like '%sc -l'.  The reason for this is to make it as easy as possible
         to process line-oriented shell output via further python commands.
         %sc is meant to provide much finer control, but requires more
         typing.
 
         3) Just like %sc -l, this is a list with special attributes:
         ::
 
           .l (or .list) : value as list.
           .n (or .nlstr): value as newline-separated string.
           .s (or .spstr): value as whitespace-separated string.
 
         This is very useful when trying to use such lists as arguments to
         system commands."""
         
         if cell is None:
             # line magic
             return self.shell.getoutput(line)
         else:
             opts,args = self.parse_options(line, '', 'out=')
             output = self.shell.getoutput(cell)
             out_name = opts.get('out', opts.get('o'))
             if out_name:
                 self.shell.user_ns[out_name] = output
             else:
                 return output
 
     system = line_cell_magic('system')(sx)
     bang = cell_magic('!')(sx)
 
     @line_magic
     def bookmark(self, parameter_s=''):
         """Manage IPython's bookmark system.
 
         %bookmark <name>       - set bookmark to current dir
         %bookmark <name> <dir> - set bookmark to <dir>
         %bookmark -l           - list all bookmarks
         %bookmark -d <name>    - remove bookmark
         %bookmark -r           - remove all bookmarks
 
         You can later on access a bookmarked folder with::
 
           %cd -b <name>
 
         or simply '%cd <name>' if there is no directory called <name> AND
         there is such a bookmark defined.
 
         Your bookmarks persist through IPython sessions, but they are
         associated with each profile."""
 
         opts,args = self.parse_options(parameter_s,'drl',mode='list')
         if len(args) > 2:
             raise UsageError("%bookmark: too many arguments")
 
         bkms = self.shell.db.get('bookmarks',{})
 
         if 'd' in opts:
             try:
                 todel = args[0]
             except IndexError as e:
                 raise UsageError(
                     "%bookmark -d: must provide a bookmark to delete") from e
             else:
                 try:
                     del bkms[todel]
                 except KeyError as e:
                     raise UsageError(
                         "%%bookmark -d: Can't delete bookmark '%s'" % todel) from e
 
         elif 'r' in opts:
             bkms = {}
         elif 'l' in opts:
             bks = sorted(bkms)
             if bks:
                 size = max(map(len, bks))
             else:
                 size = 0
             fmt = '%-'+str(size)+'s -> %s'
             print('Current bookmarks:')
             for bk in bks:
                 print(fmt % (bk, bkms[bk]))
         else:
             if not args:
                 raise UsageError("%bookmark: You must specify the bookmark name")
             elif len(args)==1:
                 bkms[args[0]] = os.getcwd()
             elif len(args)==2:
                 bkms[args[0]] = args[1]
         self.shell.db['bookmarks'] = bkms
 
     @line_magic
     def pycat(self, parameter_s=''):
         """Show a syntax-highlighted file through a pager.
 
         This magic is similar to the cat utility, but it will assume the file
         to be Python source and will show it with syntax highlighting.
 
         This magic command can either take a local filename, an url,
         an history range (see %history) or a macro as argument.
 
         If no parameter is given, prints out history of current session up to
         this point. ::
 
         %pycat myscript.py
         %pycat 7-27
         %pycat myMacro
         %pycat http://www.example.com/myscript.py
         """
         try:
             cont = self.shell.find_user_code(parameter_s, skip_encoding_cookie=False)
         except (ValueError, IOError):
             print("Error: no such file, variable, URL, history range or macro")
             return
 
         page.page(self.shell.pycolorize(source_to_unicode(cont)))
 
     @magic_arguments.magic_arguments()
     @magic_arguments.argument(
         '-a', '--append', action='store_true', default=False,
         help='Append contents of the cell to an existing file. '
              'The file will be created if it does not exist.'
     )
     @magic_arguments.argument(
         'filename', type=str,
         help='file to write'
     )
     @cell_magic
     def writefile(self, line, cell):
         """Write the contents of the cell to a file.
 
         The file will be overwritten unless the -a (--append) flag is specified.
         """
         args = magic_arguments.parse_argstring(self.writefile, line)
         if re.match(r'^(\'.*\')|(".*")$', args.filename):
             filename = os.path.expanduser(args.filename[1:-1])
         else:
             filename = os.path.expanduser(args.filename)
             
         if os.path.exists(filename):
             if args.append:
                 print("Appending to %s" % filename)
             else:
                 print("Overwriting %s" % filename)
         else:
             print("Writing %s" % filename)
         
         mode = 'a' if args.append else 'w'
         with io.open(filename, mode, encoding='utf-8') as f:
             f.write(cell)

docs/source/links.txt

@@ -1,101 +1,97 b''
 .. This (-*- rst -*-) format file contains commonly used link targets
    and name substitutions.  It may be included in many files,
    therefore it should only contain link targets and name
    substitutions.  Try grepping for "^\.\. _" to find plausible
    candidates for this list.  
 
    NOTE: this file must have an extension *opposite* to that of the main reST
    files in the manuals, so that we can include it with ".. include::"
    directives, but without triggering warnings from Sphinx for not being listed
    in any toctree.  Since IPython uses .txt for the main files, this one will
    use .rst.
    
    NOTE: reST targets are
    __not_case_sensitive__, so only one target definition is needed for
    ipython, IPython, etc.
 
    NOTE: Some of these were taken from the nipy links compendium.
 
 .. Main IPython links
 .. _ipython: https://ipython.org
 .. _`ipython manual`: https://ipython.org/documentation.html
 .. _ipython_github: http://github.com/ipython/ipython/
-.. _ipython_github_repo: http://github.com/ipython/ipython/
-.. _ipython_downloads: https://ipython.org/download.html
-.. _ipython_pypi: http://pypi.python.org/pypi/ipython
 .. _nbviewer: http://nbviewer.ipython.org
 
 .. _ZeroMQ: http://zeromq.org
 
 .. Documentation tools and related links
 .. _graphviz: http://www.graphviz.org
 .. _Sphinx: http://sphinx.pocoo.org
 .. _`Sphinx reST`: http://sphinx.pocoo.org/rest.html
 .. _sampledoc: http://matplotlib.org/sampledoc
 .. _reST: http://docutils.sourceforge.net/rst.html
 .. _docutils: http://docutils.sourceforge.net
 .. _lyx: http://www.lyx.org
-.. _pep8: http://www.python.org/dev/peps/pep-0008
-.. _numpy_coding_guide: https://github.com/numpy/numpy/blob/master/doc/HOWTO_DOCUMENT.rst.txt
+.. _pep8: https://peps.python.org/pep-0008/
 
 .. Licenses
 .. _GPL: http://www.gnu.org/licenses/gpl.html
 .. _BSD: http://www.opensource.org/licenses/bsd-license.php
 .. _LGPL: http://www.gnu.org/copyleft/lesser.html
 
 .. Other python projects
 .. _numpy: http://numpy.scipy.org
 .. _scipy: http://www.scipy.org
 .. _scipy_conference: http://conference.scipy.org
 .. _matplotlib: http://matplotlib.org
 .. _pythonxy: https://code.google.com/p/pythonxy/
 .. _ETS: http://code.enthought.com/projects/tool-suite.php
 .. _EPD: http://www.enthought.com/products/epd.php
 .. _python: http://www.python.org
 .. _mayavi: http://code.enthought.com/projects/mayavi
 .. _sympy: http://code.google.com/p/sympy
 .. _sage: http://sagemath.org
 .. _pydy: http://code.google.com/p/pydy
 .. _vpython: http://vpython.org
 .. _cython: http://cython.org
 .. _software carpentry: http://software-carpentry.org
 
 .. Not so python scientific computing tools
 .. _matlab: http://www.mathworks.com
 .. _VTK: http://vtk.org
 
 .. Other organizations
 .. _enthought: http://www.enthought.com
 .. _kitware: http://www.kitware.com
 .. _netlib: http://netlib.org
 
 .. Other tools and projects
 .. _indefero: http://www.indefero.net
 .. _git: http://git-scm.com
 .. _github: http://github.com
 .. _Markdown: http://daringfireball.net/projects/markdown/syntax
 
 .. _Running Code in the IPython Notebook: notebook_p1_
 .. _notebook_p1: http://nbviewer.ipython.org/urls/raw.github.com/ipython/ipython/1.x/examples/notebooks/Part%25201%2520-%2520Running%2520Code.ipynb
 
 .. _Basic Output: notebook_p2_
 .. _notebook_p2: http://nbviewer.ipython.org/urls/raw.github.com/ipython/ipython/1.x/examples/notebooks/Part%202%20-%20Basic%20Output.ipynb
 
 .. _Plotting with Matplotlib: notebook_p3_
 .. _notebook_p3: http://nbviewer.ipython.org/urls/raw.github.com/ipython/ipython/1.x/examples/notebooks/Part%203%20-%20Plotting%20with%20Matplotlib.ipynb
 
 .. _Markdown Cells: notebook_p4_
 .. _notebook_p4: http://nbviewer.ipython.org/urls/raw.github.com/ipython/ipython/1.x/examples/notebooks/Part%204%20-%20Markdown%20Cells.ipynb
 
 .. _Rich Display System: notebook_p5_
 .. _notebook_p5: http://nbviewer.ipython.org/urls/raw.github.com/ipython/ipython/1.x/examples/notebooks/Part%205%20-%20Rich%20Display%20System.ipynb
 
 .. _notebook_custom_display: http://nbviewer.ipython.org/urls/raw.github.com/ipython/ipython/1.x/examples/notebooks/Custom%20Display%20Logic.ipynb
 
 .. _Frontend/Kernel Model: notebook_two_proc_
 .. _notebook_two_proc: http://nbviewer.ipython.org/urls/raw.github.com/ipython/ipython/1.x/examples/notebooks/Frontend-Kernel%20Model.ipynb
 
 .. _Cell magics: notebook_cell_magics_
 .. _notebook_cell_magics: http://nbviewer.ipython.org/urls/raw.github.com/ipython/ipython/1.x/examples/notebooks/Cell%20Magics.ipynb
 
 

docs/source/whatsnew/version6.rst

@@ -1,364 +1,364 b''
 ============
  6.x Series
 ============
 
 .. _whatsnew650:
 
 IPython 6.5.0
 =============
 
 Miscellaneous bug fixes and compatibility with Python 3.7.
 
 * Autocompletion fix for modules with out ``__init__.py`` :ghpull:`11227`
 * update the ``%pastebin`` magic to use ``dpaste.com`` instead og GitHub Gist
   which now requires authentication :ghpull:`11182`
 * Fix crash with multiprocessing :ghpull:`11185`
 
 .. _whatsnew640:
 
 IPython 6.4.0
 =============
 
 Everything new in :ref:`IPython 5.7 <whatsnew570>`
 
 * Fix display object not emitting metadata :ghpull:`11106`
 * Comments failing Jedi test :ghpull:`11110` 
 
 
 .. _whatsnew631:
 
 IPython 6.3.1
 =============
 
 This is a bugfix release to switch the default completions back to IPython's
 own completion machinery. We discovered some problems with the completions
 from Jedi, including completing column names on pandas data frames.
 
 You can switch the completions source with the config option
 :configtrait:`Completer.use_jedi`.
 
 .. _whatsnew630:
 
 IPython 6.3
 ===========
 
 IPython 6.3 contains all the bug fixes and features in
 :ref:`IPython 5.6 <whatsnew560>`. In addition:
 
 * A new display class :class:`IPython.display.Code` can be used to display
   syntax highlighted code in a notebook (:ghpull:`10978`).
 * The :cellmagic:`html` magic now takes a ``--isolated`` option to put the
   content in an iframe (:ghpull:`10962`).
 * The code to find completions using the Jedi library has had various
   adjustments. This is still a work in progress, but we hope this version has
   fewer annoyances (:ghpull:`10956`, :ghpull:`10969`, :ghpull:`10999`,
   :ghpull:`11035`, :ghpull:`11063`, :ghpull:`11065`).
 * The *post* event callbacks are now always called, even when the execution failed
   (for example because of a ``SyntaxError``).
 * The execution info and result objects are now made available in the
   corresponding *pre* or *post* ``*_run_cell`` :doc:`event callbacks </config/callbacks>`
   in a backward compatible manner (:ghissue:`10774` and :ghpull:`10795`).
 * Performance with very long code cells (hundreds of lines) is greatly improved
   (:ghpull:`10898`). Further improvements are planned for IPython 7.
 
 You can see all `pull requests for the 6.3 milestone
 <https://github.com/ipython/ipython/pulls?utf8=%E2%9C%93&q=is%3Apr+milestone%3A6.3+is%3Aclosed>`__.
 
 .. _whatsnew620:
 
 IPython 6.2
 ===========
 
 IPython 6.2 contains all the bugs fixes and features :ref:`available in IPython 5.5 <whatsnew550>`,
 like built in progress bar support, and system-wide configuration
 
 The following features are specific to IPython 6.2:
 
 Function signature in completions
 ---------------------------------
 
 Terminal IPython will now show the signature of the function while completing.
 Only the currently highlighted function will show its signature on the line
 below the completer by default. This functionality is recent, so it might be
 limited; we welcome bug reports and requests for enhancements. :ghpull:`10507`
 
 Assignments return values
 -------------------------
 
 IPython can now trigger the display hook on the last assignment of cells.
 Up until 6.2 the following code wouldn't show the value of the assigned
 variable::
 
     In[1]: xyz = "something"
     # nothing shown
 
 You would have to actually make it the last statement::
 
     In [2]: xyz = "something else"
     ...   : xyz
     Out[2]: "something else"
 
 With the option ``InteractiveShell.ast_node_interactivity='last_expr_or_assign'``
 you can now do::
 
     In [2]: xyz = "something else"
     Out[2]: "something else"
 
 This option can be toggled at runtime with the ``%config`` magic, and will
 trigger on assignment ``a = 1``, augmented assignment ``+=``, ``-=``, ``|=`` ...
 as well as type annotated assignments: ``a:int = 2``.
 
 See :ghpull:`10598`
 
 Recursive Call of ipdb
 ----------------------
 
 Advanced users of the debugger can now correctly recursively enter ipdb. This is
 thanks to ``@segevfiner`` on :ghpull:`10721`.
 
 .. _whatsnew610:
 
 IPython 6.1
 ===========
 
 - Quotes in a filename are always escaped during tab-completion on non-Windows.
   :ghpull:`10069`
 
 - Variables now shadow magics in autocompletion. See :ghissue:`4877` and :ghpull:`10542`.
 
 - Added the ability to add parameters to alias_magic. For example::
 
     In [2]: %alias_magic hist history --params "-l 2" --line
     Created `%hist` as an alias for `%history -l 2`.
 
     In [3]: hist
     %alias_magic hist history --params "-l 30" --line
     %alias_magic hist history --params "-l 2" --line
 
   Previously it was only possible to have an alias attached to a single function,
   and you would have to pass in the given parameters every time::
 
     In [4]: %alias_magic hist history --line
     Created `%hist` as an alias for `%history`.
 
     In [5]: hist -l 2
     hist
     %alias_magic hist history --line
 
 - To suppress log state messages, you can now either use ``%logstart -q``, pass
   ``--LoggingMagics.quiet=True`` on the command line, or set
   ``c.LoggingMagics.quiet=True`` in your configuration file.
 
 - An additional flag ``--TerminalInteractiveShell.term_title_format`` is
   introduced to allow the user to control the format of the terminal title.  It
   is specified as a python format string, and currently the only variable it
   will format is ``{cwd}``.
 
 - ``??``/``%pinfo2`` will now show object docstrings if the source can't be retrieved. :ghpull:`10532`
 - ``IPython.display`` has gained a ``%markdown`` cell magic. :ghpull:`10563`
 - ``%config`` options can now be tab completed. :ghpull:`10555`
 - ``%config`` with no arguments are now unique and sorted. :ghpull:`10548`
 - Completion on keyword arguments does not duplicate ``=`` sign if already present. :ghpull:`10547`
 - ``%run -m <module>`` now ``<module>`` passes extra arguments to ``<module>``. :ghpull:`10546`
 - completer now understand "snake case auto complete": if ``foo_bar_kittens`` is
   a valid completion, I can type ``f_b<tab>`` will complete to it. :ghpull:`10537`
 - tracebacks are better standardized and will compress `/path/to/home` to `~`. :ghpull:`10515`
 
 The following changes were also added to IPython 5.4, see :ref:`what's new in IPython 5.4 <whatsnew540>`
 for more detail description:
 
 - ``TerminalInteractiveShell`` is configurable and can be configured to
   (re)-use the readline interface.
 
 - objects can now define a ``_repr_mimebundle_``
 
 - Execution heuristics improve for single line statements
 - ``display()`` can now return a display id to update display areas.
 
 
 .. _whatsnew600:
 
 IPython 6.0
 ===========
 
 Released April 19th, 2017
 
 IPython 6 features a major improvement in the completion machinery which is now
 capable of completing non-executed code. It is also the first version of IPython
 to stop compatibility with Python 2, which is still supported on the bugfix only
 5.x branch. Read below for a non-exhaustive list of new features.
 
 Make sure you have pip > 9.0 before upgrading.
 You should be able to update by using:
 
 .. code::
 
     pip install ipython --upgrade
 
 
 .. note::
 
     If your pip version is greater than or equal to pip 9.0.1 you will automatically get
     the most recent version of IPython compatible with your system: on Python 2 you 
     will get the latest IPython 5.x bugfix, while in Python 3
     you will get the latest 6.x stable version.
 
 New completion API and Interface
 --------------------------------
 
 The completer Completion API has seen an overhaul, and the new completer has
 plenty of improvements both from the end users of terminal IPython and for
 consumers of the API.
 
 This new API is capable of pulling completions from :any:`jedi`, thus allowing
 type inference on non-executed code. If :any:`jedi` is installed, completions like
 the following are now possible without code evaluation:
 
     >>> data = ['Number of users', 123_456]
     ... data[0].<tab>
 
 That is to say, IPython is now capable of inferring that `data[0]` is a string,
 and will suggest completions like `.capitalize`. The completion power of IPython
 will increase with new Jedi releases, and a number of bug-fixes and more completions
 are already available on the development version of :any:`jedi` if you are curious.
 
 With the help of prompt toolkit, types of completions can be shown in the
 completer interface:
 
 .. image:: ../_images/jedi_type_inference_60.png
     :alt: Jedi showing ability to do type inference
     :align: center
     :width: 400px
     :target: ../_images/jedi_type_inference_60.png
 
 The appearance of the completer is controlled by the
 ``c.TerminalInteractiveShell.display_completions`` option that will show the
 type differently depending on the value among ``'column'``, ``'multicolumn'``
 and ``'readlinelike'``
 
 The use of Jedi also fulfills a number of requests and fixes a number of bugs
 like case-insensitive completion and completion after division operator: See
 :ghpull:`10182`.
 
 Extra patches and updates will be needed to the :mod:`ipykernel` package for
 this feature to be available to other clients like Jupyter Notebook, Lab,
 Nteract, Hydrogen...
 
 The use of Jedi should be barely noticeable on recent machines, but 
 can be slower on older ones.  To tweak the performance, the amount
 of time given to Jedi to compute type inference can be adjusted with
 ``c.IPCompleter.jedi_compute_type_timeout``. The objects whose type were not
 inferred will be shown as ``<unknown>``. Jedi can also be completely deactivated
 by using the ``c.Completer.use_jedi=False`` option.
 
 
 The old ``Completer.complete()`` API is waiting deprecation and should be
 replaced replaced by ``Completer.completions()`` in the near future. Feedback on
 the current state of the API and suggestions are welcome.
 
 Python 3 only codebase
 ----------------------
 
 One of the large challenges in IPython 6.0 has been the adoption of a pure
 Python 3 codebase, which has led to upstream patches in pip,
 pypi and warehouse to make sure Python 2 systems still upgrade to the latest
 compatible Python version.
 
 We remind our Python 2 users that IPython 5 is still compatible with Python 2.7,
 still maintained and will get regular releases. Using pip 9+, upgrading IPython will
 automatically upgrade to the latest version compatible with your system.
 
 .. warning::
 
   If you are on a system using an older version of pip on Python 2, pip may
   still install IPython 6.0 on your system, and IPython will refuse to start.
   You can fix this by upgrading pip, and reinstalling ipython, or forcing pip to
   install an earlier version: ``pip install 'ipython<6'``
 
 The ability to use only Python 3 on the code base of IPython brings a number
 of advantages. Most of the newly written code make use of `optional function type
-annotation <https://www.python.org/dev/peps/pep-0484/>`_ leading to clearer code
+annotation <https://peps.python.org/pep-0484/>`_ leading to clearer code
 and better documentation.
 
 The total size of the repository has also decreased by about 1500 lines (for the 
 first time excluding the big split for 4.0). The decrease is potentially
 a bit more for the sour as some documents like this one are append only and
 are about 300 lines long.
 
 The removal of the Python2/Python3 shim layer has made the code quite a lot clearer and
 more idiomatic in a number of locations, and much friendlier to work with and
 understand. We hope to further embrace Python 3 capabilities in the next release
 cycle and introduce more of the Python 3 only idioms (yield from, kwarg only,
 general unpacking) in the IPython code base, and see if we can take advantage
 of these to improve user experience with better error messages and
 hints.
 
 
 Configurable TerminalInteractiveShell, readline interface
 ---------------------------------------------------------
 
 IPython gained a new ``c.TerminalIPythonApp.interactive_shell_class`` option
 that allows customizing the class used to start the terminal frontend. This
 should allow a user to use custom interfaces, like reviving the former readline
 interface which is now a separate package not actively maintained by the core
 team. See the project to bring back the readline interface: `rlipython
 <https://github.com/ipython/rlipython>`_.
 
 This change will be backported to the IPython 5.x series.
 
 Misc improvements
 -----------------
 
 
 - The :cellmagic:`capture` magic can now capture the result of a cell (from
   an expression on the last line), as well as printed and displayed output.
   :ghpull:`9851`.
 
 - Pressing Ctrl-Z in the terminal debugger now suspends IPython, as it already
   does in the main terminal prompt.
 
 - Autoreload can now reload ``Enum``. See :ghissue:`10232` and :ghpull:`10316`
 
 - IPython.display has gained a :any:`GeoJSON <IPython.display.GeoJSON>` object.
   :ghpull:`10288` and :ghpull:`10253`
 
 Functions Deprecated in 6.x Development cycle
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 - Loading extensions from ``ipython_extension_dir`` prints a warning that this
   location is pending deprecation. This should only affect users still having
   extensions installed with ``%install_ext`` which has been deprecated since
   IPython 4.0, and removed in 5.0. Extensions still present in
   ``ipython_extension_dir`` may shadow more recently installed versions using
   pip. It is thus recommended to clean ``ipython_extension_dir`` of any
   extension now available as a package.
 
 
 - ``IPython.utils.warn`` was deprecated in IPython 4.0, and has now been removed.
   instead of ``IPython.utils.warn`` inbuilt :any:`warnings` module is used.
 
 
 - The function `IPython.core.oinspect.py:call_tip` is unused, was marked as
   deprecated (raising a `DeprecationWarning`) and marked for later removal.
   :ghpull:`10104`
 
 Backward incompatible changes
 ------------------------------
 
 Functions Removed in 6.x Development cycle
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 The following functions have been removed in the
 development cycle marked for Milestone 6.0.
 
 - ``IPython/utils/process.py`` - ``is_cmd_found``
 - ``IPython/utils/process.py`` - ``pycmd2argv``
 
 - The `--deep-reload` flag and the corresponding options to inject `dreload` or
   `reload` into the interactive namespace have been removed. You have to
   explicitly import `reload` from `IPython.lib.deepreload` to use it.
 
 - The :magic:`profile` used to print the current IPython profile, and which
   was deprecated in IPython 2.0 does now raise a `DeprecationWarning` error when
   used. It is often confused with the :magic:`prun` and the deprecation removal
   should free up the ``profile`` name in future versions.

examples/IPython Kernel/Importing Notebooks.ipynb

@@ -1,763 +1,763 b''
 {
  "cells": [
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
     "# Importing IPython Notebooks as Modules"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
     "It is a common problem that people want to import code from IPython Notebooks.\n",
     "This is made difficult by the fact that Notebooks are not plain Python files,\n",
     "and thus cannot be imported by the regular Python machinery.\n",
     "\n",
-    "Fortunately, Python provides some fairly sophisticated [hooks](http://www.python.org/dev/peps/pep-0302/) into the import machinery,\n",
+    "Fortunately, Python provides some fairly sophisticated [hooks](https://peps.python.org/pep-0302/) into the import machinery,\n",
     "so we can actually make IPython notebooks importable without much difficulty,\n",
     "and only using public APIs."
    ]
   },
   {
    "cell_type": "code",
    "execution_count": 1,
    "metadata": {
     "collapsed": false
    },
    "outputs": [],
    "source": [
     "import io, os, sys, types"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": 2,
    "metadata": {
     "collapsed": false
    },
    "outputs": [],
    "source": [
     "import nbformat\n",
     "\n",
     "from IPython import get_ipython\n",
     "from IPython.core.interactiveshell import InteractiveShell"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
     "Import hooks typically take the form of two objects:\n",
     "\n",
     "1. a Module **Loader**, which takes a module name (e.g. `'IPython.display'`), and returns a Module\n",
     "2. a Module **Finder**, which figures out whether a module might exist, and tells Python what **Loader** to use"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": 3,
    "metadata": {
     "collapsed": false
    },
    "outputs": [],
    "source": [
     "def find_notebook(fullname, path=None):\n",
     "    \"\"\"find a notebook, given its fully qualified name and an optional path\n",
     "    \n",
     "    This turns \"foo.bar\" into \"foo/bar.ipynb\"\n",
     "    and tries turning \"Foo_Bar\" into \"Foo Bar\" if Foo_Bar\n",
     "    does not exist.\n",
     "    \"\"\"\n",
     "    name = fullname.rsplit('.', 1)[-1]\n",
     "    if not path:\n",
     "        path = ['']\n",
     "    for d in path:\n",
     "        nb_path = os.path.join(d, name + \".ipynb\")\n",
     "        if os.path.isfile(nb_path):\n",
     "            return nb_path\n",
     "        # let import Notebook_Name find \"Notebook Name.ipynb\"\n",
     "        nb_path = nb_path.replace(\"_\", \" \")\n",
     "        if os.path.isfile(nb_path):\n",
     "            return nb_path\n",
     "            "
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
     "## Notebook Loader"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
     "Here we have our Notebook Loader.\n",
     "It's actually quite simple - once we figure out the filename of the module,\n",
     "all it does is:\n",
     "\n",
     "1. load the notebook document into memory\n",
     "2. create an empty Module\n",
     "3. execute every cell in the Module namespace\n",
     "\n",
     "Since IPython cells can have extended syntax,\n",
     "the IPython transform is applied to turn each of these cells into their pure-Python counterparts before executing them.\n",
     "If all of your notebook cells are pure-Python,\n",
     "this step is unnecessary."
    ]
   },
   {
    "cell_type": "code",
    "execution_count": 4,
    "metadata": {
     "collapsed": false
    },
    "outputs": [],
    "source": [
     "class NotebookLoader(object):\n",
     "    \"\"\"Module Loader for IPython Notebooks\"\"\"\n",
     "    def __init__(self, path=None):\n",
     "        self.shell = InteractiveShell.instance()\n",
     "        self.path = path\n",
     "    \n",
     "    def load_module(self, fullname):\n",
     "        \"\"\"import a notebook as a module\"\"\"\n",
     "        path = find_notebook(fullname, self.path)\n",
     "        \n",
     "        print (\"importing notebook from %s\" % path)\n",
     "                                       \n",
     "        # load the notebook object\n",
     "        nb = nbformat.read(path, as_version=4)\n",
     "        \n",
     "        \n",
     "        # create the module and add it to sys.modules\n",
     "        # if name in sys.modules:\n",
     "        #    return sys.modules[name]\n",
     "        mod = types.ModuleType(fullname)\n",
     "        mod.__file__ = path\n",
     "        mod.__loader__ = self\n",
     "        mod.__dict__['get_ipython'] = get_ipython\n",
     "        sys.modules[fullname] = mod\n",
     "        \n",
     "        # extra work to ensure that magics that would affect the user_ns\n",
     "        # actually affect the notebook module's ns\n",
     "        save_user_ns = self.shell.user_ns\n",
     "        self.shell.user_ns = mod.__dict__\n",
     "        \n",
     "        try:\n",
     "          for cell in nb.cells:\n",
     "            if cell.cell_type == 'code':\n",
     "                # transform the input to executable Python\n",
     "                code = self.shell.input_transformer_manager.transform_cell(cell.source)\n",
     "                # run the code in themodule\n",
     "                exec(code, mod.__dict__)\n",
     "        finally:\n",
     "            self.shell.user_ns = save_user_ns\n",
     "        return mod\n"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
     "## The Module Finder"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
     "The finder is a simple object that tells you whether a name can be imported,\n",
     "and returns the appropriate loader.\n",
     "All this one does is check, when you do:\n",
     "\n",
     "```python\n",
     "import mynotebook\n",
     "```\n",
     "\n",
     "it checks whether `mynotebook.ipynb` exists.\n",
     "If a notebook is found, then it returns a NotebookLoader.\n",
     "\n",
     "Any extra logic is just for resolving paths within packages."
    ]
   },
   {
    "cell_type": "code",
    "execution_count": 5,
    "metadata": {
     "collapsed": false
    },
    "outputs": [],
    "source": [
     "class NotebookFinder(object):\n",
     "    \"\"\"Module finder that locates IPython Notebooks\"\"\"\n",
     "    def __init__(self):\n",
     "        self.loaders = {}\n",
     "    \n",
     "    def find_module(self, fullname, path=None):\n",
     "        nb_path = find_notebook(fullname, path)\n",
     "        if not nb_path:\n",
     "            return\n",
     "        \n",
     "        key = path\n",
     "        if path:\n",
     "            # lists aren't hashable\n",
     "            key = os.path.sep.join(path)\n",
     "        \n",
     "        if key not in self.loaders:\n",
     "            self.loaders[key] = NotebookLoader(path)\n",
     "        return self.loaders[key]\n"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
     "## Register the hook"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
     "Now we register the `NotebookFinder` with `sys.meta_path`"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": 6,
    "metadata": {
     "collapsed": false
    },
    "outputs": [],
    "source": [
     "sys.meta_path.append(NotebookFinder())"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
     "After this point, my notebooks should be importable.\n",
     "\n",
     "Let's look at what we have in the CWD:"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": 7,
    "metadata": {
     "collapsed": false
    },
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
       "__init__.py       \u001b[34m__pycache__\u001b[m\u001b[m/      mynotebook.ipynb  \u001b[34mnbs\u001b[m\u001b[m/\r\n"
      ]
     }
    ],
    "source": [
     "ls nbpackage"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
     "So I should be able to `import nbimp.mynotebook`.\n"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
     "### Aside: displaying notebooks"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
     "Here is some simple code to display the contents of a notebook\n",
     "with syntax highlighting, etc."
    ]
   },
   {
    "cell_type": "code",
    "execution_count": 8,
    "metadata": {
     "collapsed": false
    },
    "outputs": [
     {
      "data": {
       "text/html": [
        "\n",
        "<style type='text/css'>\n",
        ".hll { background-color: #ffffcc }\n",
        ".c { color: #408080; font-style: italic } /* Comment */\n",
        ".err { border: 1px solid #FF0000 } /* Error */\n",
        ".k { color: #008000; font-weight: bold } /* Keyword */\n",
        ".o { color: #666666 } /* Operator */\n",
        ".cm { color: #408080; font-style: italic } /* Comment.Multiline */\n",
        ".cp { color: #BC7A00 } /* Comment.Preproc */\n",
        ".c1 { color: #408080; font-style: italic } /* Comment.Single */\n",
        ".cs { color: #408080; font-style: italic } /* Comment.Special */\n",
        ".gd { color: #A00000 } /* Generic.Deleted */\n",
        ".ge { font-style: italic } /* Generic.Emph */\n",
        ".gr { color: #FF0000 } /* Generic.Error */\n",
        ".gh { color: #000080; font-weight: bold } /* Generic.Heading */\n",
        ".gi { color: #00A000 } /* Generic.Inserted */\n",
        ".go { color: #888888 } /* Generic.Output */\n",
        ".gp { color: #000080; font-weight: bold } /* Generic.Prompt */\n",
        ".gs { font-weight: bold } /* Generic.Strong */\n",
        ".gu { color: #800080; font-weight: bold } /* Generic.Subheading */\n",
        ".gt { color: #0044DD } /* Generic.Traceback */\n",
        ".kc { color: #008000; font-weight: bold } /* Keyword.Constant */\n",
        ".kd { color: #008000; font-weight: bold } /* Keyword.Declaration */\n",
        ".kn { color: #008000; font-weight: bold } /* Keyword.Namespace */\n",
        ".kp { color: #008000 } /* Keyword.Pseudo */\n",
        ".kr { color: #008000; font-weight: bold } /* Keyword.Reserved */\n",
        ".kt { color: #B00040 } /* Keyword.Type */\n",
        ".m { color: #666666 } /* Literal.Number */\n",
        ".s { color: #BA2121 } /* Literal.String */\n",
        ".na { color: #7D9029 } /* Name.Attribute */\n",
        ".nb { color: #008000 } /* Name.Builtin */\n",
        ".nc { color: #0000FF; font-weight: bold } /* Name.Class */\n",
        ".no { color: #880000 } /* Name.Constant */\n",
        ".nd { color: #AA22FF } /* Name.Decorator */\n",
        ".ni { color: #999999; font-weight: bold } /* Name.Entity */\n",
        ".ne { color: #D2413A; font-weight: bold } /* Name.Exception */\n",
        ".nf { color: #0000FF } /* Name.Function */\n",
        ".nl { color: #A0A000 } /* Name.Label */\n",
        ".nn { color: #0000FF; font-weight: bold } /* Name.Namespace */\n",
        ".nt { color: #008000; font-weight: bold } /* Name.Tag */\n",
        ".nv { color: #19177C } /* Name.Variable */\n",
        ".ow { color: #AA22FF; font-weight: bold } /* Operator.Word */\n",
        ".w { color: #bbbbbb } /* Text.Whitespace */\n",
        ".mb { color: #666666 } /* Literal.Number.Bin */\n",
        ".mf { color: #666666 } /* Literal.Number.Float */\n",
        ".mh { color: #666666 } /* Literal.Number.Hex */\n",
        ".mi { color: #666666 } /* Literal.Number.Integer */\n",
        ".mo { color: #666666 } /* Literal.Number.Oct */\n",
        ".sb { color: #BA2121 } /* Literal.String.Backtick */\n",
        ".sc { color: #BA2121 } /* Literal.String.Char */\n",
        ".sd { color: #BA2121; font-style: italic } /* Literal.String.Doc */\n",
        ".s2 { color: #BA2121 } /* Literal.String.Double */\n",
        ".se { color: #BB6622; font-weight: bold } /* Literal.String.Escape */\n",
        ".sh { color: #BA2121 } /* Literal.String.Heredoc */\n",
        ".si { color: #BB6688; font-weight: bold } /* Literal.String.Interpol */\n",
        ".sx { color: #008000 } /* Literal.String.Other */\n",
        ".sr { color: #BB6688 } /* Literal.String.Regex */\n",
        ".s1 { color: #BA2121 } /* Literal.String.Single */\n",
        ".ss { color: #19177C } /* Literal.String.Symbol */\n",
        ".bp { color: #008000 } /* Name.Builtin.Pseudo */\n",
        ".vc { color: #19177C } /* Name.Variable.Class */\n",
        ".vg { color: #19177C } /* Name.Variable.Global */\n",
        ".vi { color: #19177C } /* Name.Variable.Instance */\n",
        ".il { color: #666666 } /* Literal.Number.Integer.Long */\n",
        "</style>\n"
       ],
       "text/plain": [
        "<IPython.core.display.HTML object>"
       ]
      },
      "metadata": {},
      "output_type": "display_data"
     }
    ],
    "source": [
     "from pygments import highlight\n",
     "from pygments.lexers import PythonLexer\n",
     "from pygments.formatters import HtmlFormatter\n",
     "\n",
     "from IPython.display import display, HTML\n",
     "\n",
     "formatter = HtmlFormatter()\n",
     "lexer = PythonLexer()\n",
     "\n",
     "# publish the CSS for pygments highlighting\n",
     "display(HTML(\"\"\"\n",
     "<style type='text/css'>\n",
     "%s\n",
     "</style>\n",
     "\"\"\" % formatter.get_style_defs()\n",
     "))"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": 9,
    "metadata": {
     "collapsed": false
    },
    "outputs": [
     {
      "data": {
       "text/html": [
        "<h4>markdown cell</h4>\n",
        "<pre># My Notebook</pre>\n",
        "<h4>code cell</h4>\n",
        "<div class=\"highlight\"><pre><span class=\"k\">def</span> <span class=\"nf\">foo</span><span class=\"p\">():</span>\n",
        "    <span class=\"k\">return</span> <span class=\"s\">&quot;foo&quot;</span>\n",
        "</pre></div>\n",
        "\n",
        "<h4>code cell</h4>\n",
        "<div class=\"highlight\"><pre><span class=\"k\">def</span> <span class=\"nf\">has_ip_syntax</span><span class=\"p\">():</span>\n",
        "    <span class=\"n\">listing</span> <span class=\"o\">=</span> <span class=\"err\">!</span><span class=\"n\">ls</span>\n",
        "    <span class=\"k\">return</span> <span class=\"n\">listing</span>\n",
        "</pre></div>\n",
        "\n",
        "<h4>code cell</h4>\n",
        "<div class=\"highlight\"><pre><span class=\"k\">def</span> <span class=\"nf\">whatsmyname</span><span class=\"p\">():</span>\n",
        "    <span class=\"k\">return</span> <span class=\"n\">__name__</span>\n",
        "</pre></div>\n"
       ],
       "text/plain": [
        "<IPython.core.display.HTML object>"
       ]
      },
      "metadata": {},
      "output_type": "display_data"
     }
    ],
    "source": [
     "def show_notebook(fname):\n",
     "    \"\"\"display a short summary of the cells of a notebook\"\"\"\n",
     "    nb = nbformat.read(fname, as_version=4)\n",
     "    html = []\n",
     "    for cell in nb.cells:\n",
     "        html.append(\"<h4>%s cell</h4>\" % cell.cell_type)\n",
     "        if cell.cell_type == 'code':\n",
     "            html.append(highlight(cell.source, lexer, formatter))\n",
     "        else:\n",
     "            html.append(\"<pre>%s</pre>\" % cell.source)\n",
     "    display(HTML('\\n'.join(html)))\n",
     "\n",
     "show_notebook(os.path.join(\"nbpackage\", \"mynotebook.ipynb\"))"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
     "So my notebook has a heading cell and some code cells,\n",
     "one of which contains some IPython syntax.\n",
     "\n",
     "Let's see what happens when we import it"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": 10,
    "metadata": {
     "collapsed": false
    },
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
       "importing notebook from /Users/minrk/dev/ip/mine/examples/IPython Kernel/nbpackage/mynotebook.ipynb\n"
      ]
     }
    ],
    "source": [
     "from nbpackage import mynotebook"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
     "Hooray, it imported!  Does it work?"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": 11,
    "metadata": {
     "collapsed": false
    },
    "outputs": [
     {
      "data": {
       "text/plain": [
        "'foo'"
       ]
      },
      "execution_count": 11,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
    "source": [
     "mynotebook.foo()"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
     "Hooray again!\n",
     "\n",
     "Even the function that contains IPython syntax works:"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": 12,
    "metadata": {
     "collapsed": false
    },
    "outputs": [
     {
      "data": {
       "text/plain": [
        "['Animations Using clear_output.ipynb',\n",
        " 'Background Jobs.ipynb',\n",
        " 'Beyond Plain Python.ipynb',\n",
        " 'Capturing Output.ipynb',\n",
        " 'Cell Magics.ipynb',\n",
        " 'Custom Display Logic.ipynb',\n",
        " 'Importing Notebooks.ipynb',\n",
        " 'Index.ipynb',\n",
        " 'Plotting in the Notebook.ipynb',\n",
        " 'Raw Input in the Notebook.ipynb',\n",
        " 'Rich Output.ipynb',\n",
        " 'Script Magics.ipynb',\n",
        " 'SymPy.ipynb',\n",
        " 'Terminal Usage.ipynb',\n",
        " 'Third Party Rich Output.ipynb',\n",
        " 'Trapezoid Rule.ipynb',\n",
        " 'Working With External Code.ipynb',\n",
        " '__pycache__',\n",
        " 'data',\n",
        " 'example-demo.py',\n",
        " 'gui',\n",
        " 'ipython-completion.bash',\n",
        " 'ipython-get-history.py',\n",
        " 'ipython.desktop',\n",
        " 'nbpackage']"
       ]
      },
      "execution_count": 12,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
    "source": [
     "mynotebook.has_ip_syntax()"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
     "## Notebooks in packages"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
     "We also have a notebook inside the `nb` package,\n",
     "so let's make sure that works as well."
    ]
   },
   {
    "cell_type": "code",
    "execution_count": 13,
    "metadata": {
     "collapsed": false
    },
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
       "__init__.py  \u001b[34m__pycache__\u001b[m\u001b[m/ other.ipynb\r\n"
      ]
     }
    ],
    "source": [
     "ls nbpackage/nbs"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
     "Note that the `__init__.py` is necessary for `nb` to be considered a package,\n",
     "just like usual."
    ]
   },
   {
    "cell_type": "code",
    "execution_count": 14,
    "metadata": {
     "collapsed": false
    },
    "outputs": [
     {
      "data": {
       "text/html": [
        "<h4>markdown cell</h4>\n",
        "<pre>This notebook just defines `bar`</pre>\n",
        "<h4>code cell</h4>\n",
        "<div class=\"highlight\"><pre><span class=\"k\">def</span> <span class=\"nf\">bar</span><span class=\"p\">(</span><span class=\"n\">x</span><span class=\"p\">):</span>\n",
        "    <span class=\"k\">return</span> <span class=\"s\">&quot;bar&quot;</span> <span class=\"o\">*</span> <span class=\"n\">x</span>\n",
        "</pre></div>\n"
       ],
       "text/plain": [
        "<IPython.core.display.HTML object>"
       ]
      },
      "metadata": {},
      "output_type": "display_data"
     }
    ],
    "source": [
     "show_notebook(os.path.join(\"nbpackage\", \"nbs\", \"other.ipynb\"))"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": 15,
    "metadata": {
     "collapsed": false
    },
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
       "importing notebook from /Users/minrk/dev/ip/mine/examples/IPython Kernel/nbpackage/nbs/other.ipynb\n"
      ]
     },
     {
      "data": {
       "text/plain": [
        "'barbarbarbarbar'"
       ]
      },
      "execution_count": 15,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
    "source": [
     "from nbpackage.nbs import other\n",
     "other.bar(5)"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
     "So now we have importable notebooks, from both the local directory and inside packages.\n",
     "\n",
     "I can even put a notebook inside IPython, to further demonstrate that this is working properly:"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": 16,
    "metadata": {
     "collapsed": false
    },
    "outputs": [
     {
      "data": {
       "text/plain": [
        "'/Users/minrk/dev/ip/mine/IPython/utils/inside_ipython.ipynb'"
       ]
      },
      "execution_count": 16,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
    "source": [
     "import shutil\n",
     "from IPython.paths import get_ipython_package_dir\n",
     "\n",
     "utils = os.path.join(get_ipython_package_dir(), 'utils')\n",
     "shutil.copy(os.path.join(\"nbpackage\", \"mynotebook.ipynb\"),\n",
     "            os.path.join(utils, \"inside_ipython.ipynb\")\n",
     ")"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
     "and import the notebook from `IPython.utils`"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": 17,
    "metadata": {
     "collapsed": false
    },
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
       "importing notebook from /Users/minrk/dev/ip/mine/IPython/utils/inside_ipython.ipynb\n"
      ]
     },
     {
      "data": {
       "text/plain": [
        "'IPython.utils.inside_ipython'"
       ]
      },
      "execution_count": 17,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
    "source": [
     "from IPython.utils import inside_ipython\n",
     "inside_ipython.whatsmyname()"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
     "This approach can even import functions and classes that are defined in a notebook using the `%%cython` magic."
    ]
   }
  ],
  "metadata": {
   "gist_id": "6011986",
   "kernelspec": {
    "display_name": "Python 3",
    "language": "python",
    "name": "python3"
   },
   "language_info": {
    "codemirror_mode": {
     "name": "ipython",
     "version": 3
    },
    "file_extension": ".py",
    "mimetype": "text/x-python",
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
    "version": "3.4.3"
   }
  },
  "nbformat": 4,
  "nbformat_minor": 0
 }