Merge pull request #1073 from takluyver/storemagic-plugin

* %store is now a plugin
* adds a 'default_extensions' list for extensions that should always be loaded.

IPython/core/shellapp.py

@@ -1,291 +1,292 b''
 # encoding: utf-8
 """
 A mixin for :class:`~IPython.core.application.Application` classes that
 launch InteractiveShell instances, load extensions, etc.
 
 Authors
 -------
 
 * Min Ragan-Kelley
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 from __future__ import absolute_import
 
 import glob
 import os
 import sys
 
 from IPython.config.application import boolean_flag
 from IPython.config.configurable import Configurable
 from IPython.config.loader import Config
 from IPython.utils import py3compat
 from IPython.utils.path import filefind
 from IPython.utils.traitlets import Unicode, Instance, List, Bool
 
 #-----------------------------------------------------------------------------
 # Aliases and Flags
 #-----------------------------------------------------------------------------
 
 shell_flags = {}
 
 addflag = lambda *args: shell_flags.update(boolean_flag(*args))
 addflag('autoindent', 'InteractiveShell.autoindent',
         'Turn on autoindenting.', 'Turn off autoindenting.'
 )
 addflag('automagic', 'InteractiveShell.automagic',
         """Turn on the auto calling of magic commands. Type %%magic at the
         IPython  prompt  for  more information.""",
         'Turn off the auto calling of magic commands.'
 )
 addflag('pdb', 'InteractiveShell.pdb',
     "Enable auto calling the pdb debugger after every exception.",
     "Disable auto calling the pdb debugger after every exception."
 )
 # pydb flag doesn't do any config, as core.debugger switches on import,
 # which is before parsing.  This just allows the flag to be passed.
 shell_flags.update(dict(
     pydb = ({},
         """"Use the third party 'pydb' package as debugger, instead of pdb.
         Requires that pydb is installed."""
     )
 ))
 addflag('pprint', 'PlainTextFormatter.pprint',
     "Enable auto pretty printing of results.",
     "Disable auto auto pretty printing of results."
 )
 addflag('color-info', 'InteractiveShell.color_info',
     """IPython can display information about objects via a set of func-
     tions, and optionally can use colors for this, syntax highlighting
     source code and various other elements.  However, because this
     information is passed through a pager (like 'less') and many pagers get
     confused with color codes, this option is off by default.  You can test
     it and turn it on permanently in your ipython_config.py file if it
     works for you.  Test it and turn it on permanently if it works with
     your system.  The magic function %%color_info allows you to toggle this
     interactively for testing.""",
     "Disable using colors for info related things."
 )
 addflag('deep-reload', 'InteractiveShell.deep_reload',
     """Enable deep (recursive) reloading by default. IPython can use the
     deep_reload module which reloads changes in modules recursively (it
     replaces the reload() function, so you don't need to change anything to
     use it). deep_reload() forces a full reload of modules whose code may
     have changed, which the default reload() function does not.  When
     deep_reload is off, IPython will use the normal reload(), but
     deep_reload will still be available as dreload(). This feature is off
     by default [which means that you have both normal reload() and
     dreload()].""",
     "Disable deep (recursive) reloading by default."
 )
 nosep_config = Config()
 nosep_config.InteractiveShell.separate_in = ''
 nosep_config.InteractiveShell.separate_out = ''
 nosep_config.InteractiveShell.separate_out2 = ''
 
 shell_flags['nosep']=(nosep_config, "Eliminate all spacing between prompts.")
 
 
 # it's possible we don't want short aliases for *all* of these:
 shell_aliases = dict(
     autocall='InteractiveShell.autocall',
     colors='InteractiveShell.colors',
     logfile='InteractiveShell.logfile',
     logappend='InteractiveShell.logappend',
     c='InteractiveShellApp.code_to_run',
     ext='InteractiveShellApp.extra_extension',
 )
 shell_aliases['cache-size'] = 'InteractiveShell.cache_size'
 
 #-----------------------------------------------------------------------------
 # Main classes and functions
 #-----------------------------------------------------------------------------
 
 class InteractiveShellApp(Configurable):
     """A Mixin for applications that start InteractiveShell instances.
     
     Provides configurables for loading extensions and executing files
     as part of configuring a Shell environment.
     
     Provides init_extensions() and init_code() methods, to be called
     after init_shell(), which must be implemented by subclasses.
     """
     extensions = List(Unicode, config=True,
         help="A list of dotted module names of IPython extensions to load."
     )
     extra_extension = Unicode('', config=True,
         help="dotted module name of an IPython extension to load."
     )
     def _extra_extension_changed(self, name, old, new):
         if new:
             # add to self.extensions
             self.extensions.append(new)
 
+    # Extensions that are always loaded (not configurable)
+    default_extensions = List(Unicode, [u'storemagic'], config=False)
+
     exec_files = List(Unicode, config=True,
         help="""List of files to run at IPython startup."""
     )
     file_to_run = Unicode('', config=True,
         help="""A file to be run""")
 
     exec_lines = List(Unicode, config=True,
         help="""lines of code to run at IPython startup."""
     )
     code_to_run = Unicode('', config=True,
         help="Execute the given command string."
     )
     pylab_import_all = Bool(True, config=True,
         help="""If true, an 'import *' is done from numpy and pylab,
         when using pylab"""
     )
     shell = Instance('IPython.core.interactiveshell.InteractiveShellABC')
 
     def init_shell(self):
         raise NotImplementedError("Override in subclasses")
     
     def init_extensions(self):
         """Load all IPython extensions in IPythonApp.extensions.
 
         This uses the :meth:`ExtensionManager.load_extensions` to load all
         the extensions listed in ``self.extensions``.
         """
-        if not self.extensions:
-            return
         try:
             self.log.debug("Loading IPython extensions...")
-            extensions = self.extensions
+            extensions = self.default_extensions + self.extensions
             for ext in extensions:
                 try:
                     self.log.info("Loading IPython extension: %s" % ext)
                     self.shell.extension_manager.load_extension(ext)
                 except:
                     self.log.warn("Error in loading extension: %s" % ext +
                         "\nCheck your config files in %s" % self.profile_dir.location
                     )
                     self.shell.showtraceback()
         except:
             self.log.warn("Unknown error in loading extensions:")
             self.shell.showtraceback()
 
     def init_code(self):
         """run the pre-flight code, specified via exec_lines"""
         self._run_startup_files()
         self._run_exec_lines()
         self._run_exec_files()
         self._run_cmd_line_code()
         
         # Hide variables defined here from %who etc.
         self.shell.user_ns_hidden.update(self.shell.user_ns)
 
     def _run_exec_lines(self):
         """Run lines of code in IPythonApp.exec_lines in the user's namespace."""
         if not self.exec_lines:
             return
         try:
             self.log.debug("Running code from IPythonApp.exec_lines...")
             for line in self.exec_lines:
                 try:
                     self.log.info("Running code in user namespace: %s" %
                                   line)
                     self.shell.run_cell(line, store_history=False)
                 except:
                     self.log.warn("Error in executing line in user "
                                   "namespace: %s" % line)
                     self.shell.showtraceback()
         except:
             self.log.warn("Unknown error in handling IPythonApp.exec_lines:")
             self.shell.showtraceback()
 
     def _exec_file(self, fname):
         try:
             full_filename = filefind(fname, [u'.', self.ipython_dir])
         except IOError as e:
             self.log.warn("File not found: %r"%fname)
             return
         # Make sure that the running script gets a proper sys.argv as if it
         # were run from a system shell.
         save_argv = sys.argv
         sys.argv = [full_filename] + self.extra_args[1:]
         # protect sys.argv from potential unicode strings on Python 2:
         if not py3compat.PY3:
             sys.argv = [ py3compat.cast_bytes(a) for a in sys.argv ]
         try:
             if os.path.isfile(full_filename):
                 if full_filename.endswith('.ipy'):
                     self.log.info("Running file in user namespace: %s" %
                                   full_filename)
                     self.shell.safe_execfile_ipy(full_filename)
                 else:
                     # default to python, even without extension
                     self.log.info("Running file in user namespace: %s" %
                                   full_filename)
                     # Ensure that __file__ is always defined to match Python behavior
                     self.shell.user_ns['__file__'] = fname
                     try:
                         self.shell.safe_execfile(full_filename, self.shell.user_ns)
                     finally:
                         del self.shell.user_ns['__file__']
         finally:
             sys.argv = save_argv
 
     def _run_startup_files(self):
         """Run files from profile startup directory"""
         startup_dir = self.profile_dir.startup_dir
         startup_files = glob.glob(os.path.join(startup_dir, '*.py'))
         startup_files += glob.glob(os.path.join(startup_dir, '*.ipy'))
         if not startup_files:
             return
         
         self.log.debug("Running startup files from %s...", startup_dir)
         try:
             for fname in sorted(startup_files):
                 self._exec_file(fname)
         except:
             self.log.warn("Unknown error in handling startup files:")
             self.shell.showtraceback()
 
     def _run_exec_files(self):
         """Run files from IPythonApp.exec_files"""
         if not self.exec_files:
             return
 
         self.log.debug("Running files in IPythonApp.exec_files...")
         try:
             for fname in self.exec_files:
                 self._exec_file(fname)
         except:
             self.log.warn("Unknown error in handling IPythonApp.exec_files:")
             self.shell.showtraceback()
 
     def _run_cmd_line_code(self):
         """Run code or file specified at the command-line"""
         if self.code_to_run:
             line = self.code_to_run
             try:
                 self.log.info("Running code given at command line (c=): %s" %
                               line)
                 self.shell.run_cell(line, store_history=False)
             except:
                 self.log.warn("Error in executing line in user namespace: %s" %
                               line)
                 self.shell.showtraceback()
 
         # Like Python itself, ignore the second if the first of these is present
         elif self.file_to_run:
             fname = self.file_to_run
             try:
                 self._exec_file(fname)
             except:
                 self.log.warn("Error in executing file in user namespace: %s" %
                               fname)
                 self.shell.showtraceback()
 

IPython/extensions/storemagic.py

@@ -1,188 +1,205 b''
 # -*- coding: utf-8 -*-
 """
 %store magic for lightweight persistence.
 
-Stores variables, aliases and macros in IPython's database. Stored values will
-be automatically restored whenever the extension is loaded.
+Stores variables, aliases and macros in IPython's database.
 
-To enable this functionality, list it in your default profile
-`ipython_config.py` file::
+To automatically restore stored variables at startup, add this to your
+:file:`ipython_config.py` file::
 
-  c.InteractiveShellApp.extensions = ['storemagic']
-
-Or to use it temporarily, run this in your IPython session::
-
-  %load_ext storemagic
+  c.StoreMagic.autorestore = True
 
 """
 
 from IPython.core.error import TryNext, UsageError
+from IPython.core.plugin import Plugin
+from IPython.testing.skipdoctest import skip_doctest
 from IPython.utils import pickleshare
+from IPython.utils.traitlets import Bool, Instance
 
 import inspect,pickle,os,sys,textwrap
 from IPython.core.fakemodule import FakeModule
 
 def restore_aliases(ip):
     staliases = ip.db.get('stored_aliases', {})
     for k,v in staliases.items():
         #print "restore alias",k,v # dbg
         #self.alias_table[k] = v
         ip.alias_manager.define_alias(k,v)
 
 
 def refresh_variables(ip):
     db = ip.db
     for key in db.keys('autorestore/*'):
         # strip autorestore
         justkey = os.path.basename(key)
         try:
             obj = db[key]
         except KeyError:
             print "Unable to restore variable '%s', ignoring (use %%store -d to forget!)" % justkey
             print "The error was:",sys.exc_info()[0]
         else:
             #print "restored",justkey,"=",obj #dbg
             ip.user_ns[justkey] = obj
 
 
 def restore_dhist(ip):
     ip.user_ns['_dh'] = ip.db.get('dhist',[])
 
 def restore_data(ip):
     refresh_variables(ip)
     restore_aliases(ip)
     restore_dhist(ip)
 
+@skip_doctest
 def magic_store(self, parameter_s=''):
     """Lightweight persistence for python variables.
 
     Example::
 
       In [1]: l = ['hello',10,'world']
       In [2]: %store l
       In [3]: exit
 
       (IPython session is closed and started again...)
 
       ville@badger:~$ ipython
       In [1]: l
       Out[1]: ['hello', 10, 'world']
 
     Usage:
 
     * ``%store``          - Show list of all variables and their current values
     * ``%store spam``     - Store the *current* value of the variable spam to disk
     * ``%store -d spam``  - Remove the variable and its value from storage
     * ``%store -z``       - Remove all variables from storage
     * ``%store -r``       - Refresh all variables from store (delete current vals)
     * ``%store foo >a.txt``  - Store value of foo to new file a.txt
     * ``%store foo >>a.txt`` - Append value of foo to file a.txt
 
     It should be noted that if you change the value of a variable, you
     need to %store it again if you want to persist the new value.
 
     Note also that the variables will need to be pickleable; most basic
     python types can be safely %store'd.
 
     Also aliases can be %store'd across sessions.
     """
 
     opts,argsl = self.parse_options(parameter_s,'drz',mode='string')
     args = argsl.split(None,1)
     ip = self.shell
     db = ip.db
     # delete
     if opts.has_key('d'):
         try:
             todel = args[0]
         except IndexError:
             raise UsageError('You must provide the variable to forget')
         else:
             try:
                 del db['autorestore/' + todel]
             except:
                 raise UsageError("Can't delete variable '%s'" % todel)
     # reset
     elif opts.has_key('z'):
         for k in db.keys('autorestore/*'):
             del db[k]
 
     elif opts.has_key('r'):
         refresh_variables(ip)
 
 
     # run without arguments -> list variables & values
     elif not args:
         vars = self.db.keys('autorestore/*')
         vars.sort()
         if vars:
             size = max(map(len,vars))
         else:
             size = 0
 
         print 'Stored variables and their in-db values:'
         fmt = '%-'+str(size)+'s -> %s'
         get = db.get
         for var in vars:
             justkey = os.path.basename(var)
             # print 30 first characters from every var
             print fmt % (justkey,repr(get(var,'<unavailable>'))[:50])
 
     # default action - store the variable
     else:
         # %store foo >file.txt or >>file.txt
         if len(args) > 1 and args[1].startswith('>'):
             fnam = os.path.expanduser(args[1].lstrip('>').lstrip())
             if args[1].startswith('>>'):
                 fil = open(fnam,'a')
             else:
                 fil = open(fnam,'w')
             obj = ip.ev(args[0])
             print "Writing '%s' (%s) to file '%s'." % (args[0],
               obj.__class__.__name__, fnam)
 
 
             if not isinstance (obj,basestring):
                 from pprint import pprint
                 pprint(obj,fil)
             else:
                 fil.write(obj)
                 if not obj.endswith('\n'):
                     fil.write('\n')
 
             fil.close()
             return
 
         # %store foo
         try:
             obj = ip.user_ns[args[0]]
         except KeyError:
             # it might be an alias
             # This needs to be refactored to use the new AliasManager stuff.
             if args[0] in self.alias_manager:
                 name = args[0]
                 nargs, cmd = self.alias_manager.alias_table[ name ]
                 staliases = db.get('stored_aliases',{})
                 staliases[ name ] = cmd
                 db['stored_aliases'] = staliases
                 print "Alias stored: %s (%s)" % (name, cmd)
                 return
             else:
                 raise UsageError("Unknown variable '%s'" % args[0])
 
         else:
             if isinstance(inspect.getmodule(obj), FakeModule):
                 print textwrap.dedent("""\
                 Warning:%s is %s
                 Proper storage of interactively declared classes (or instances
                 of those classes) is not possible! Only instances
                 of classes in real modules on file system can be %%store'd.
                 """ % (args[0], obj) )
                 return
             #pickled = pickle.dumps(obj)
             self.db[ 'autorestore/' + args[0] ] = obj
             print "Stored '%s' (%s)" % (args[0], obj.__class__.__name__)
 
+
+class StoreMagic(Plugin):
+    shell = Instance('IPython.core.interactiveshell.InteractiveShellABC')
+    autorestore = Bool(False, config=True)
+    
+    def __init__(self, shell, config):
+        super(StoreMagic, self).__init__(shell=shell, config=config)
+        shell.define_magic('store', magic_store)
+        
+        if self.autorestore:
+            restore_data(shell)
+
+_loaded = False
+
 def load_ipython_extension(ip):
-    ip.define_magic('store', magic_store)
-    restore_data(ip)
+    """Load the extension in IPython."""
+    global _loaded
+    if not _loaded:
+        plugin = StoreMagic(shell=ip, config=ip.config)
+        ip.plugin_manager.register_plugin('storemagic', plugin)
+        _loaded = True

docs/source/whatsnew/development.txt

@@ -1,154 +1,155 b''
 ================================================
 Development version
 ================================================
 
 The changes listed here are a brief summary of the substantial work on IPython
 since the 0.11.x release series. For more details, please consult the actual
 source.
 
 Main `ipython` branch
 =====================
 
 
 New features
 ------------
 
 .. Expand on this:
 * **HTML Notebook**: A powerful new interface puts IPython in your browser. You
   can start it with the command ``ipython notebook``. See :ref:`the Notebook
   docs <htmlnotebook>` for technical details.
 
 * **Tabbed QtConsole**: The QtConsole now supports starting multiple kernels in
   tabs, and has a menubar, so it looks and behaves more like a real application.
   Keyboard enthusiasts can disable the menubar with ctrl-shift-M (:ghpull:`887`).
 
 * **Python 3 compatibility**: IPython can now be installed from a single
   codebase on Python 2 and Python 3. The installation process for Python 3
   automatically runs 2to3. The same 'default' profile is now used for
   Python 2 and 3 (the previous version had a separate 'python3' profile).
 
 * **PyPy support**: The terminal interface to IPython now runs under
   `PyPy <http://pypy.org/>`_.
 
 * **SSH Tunnels**: In 0.11, the :mod:`IPython.parallel` Client could tunnel its
   connections to the Controller via ssh. Now, the QtConsole :ref:`supports
   <ssh_tunnels>` ssh tunneling, as do parallel engines.
 
 * **relaxed command-line parsing**: 0.11 was released with overly-strict
   command-line parsing, preventing the ability to specify arguments with spaces,
   e.g. ``ipython --pylab qt`` or ``ipython -c "print 'hi'"``. This has
   been fixed, by using argparse. The new parsing is a strict superset of 0.11, so
   any commands in 0.11 should still work in 0.12.
 
 * **HistoryAccessor**: The :class:`~IPython.core.history.HistoryManager` class for
   interacting with your IPython SQLite history database has been split, adding
   a parent :class:`~IPython.core.history.HistoryAccessor` class, so that users can
   write code to access and search their IPython history without being in an IPython
   session (:ghpull:`824`).
 
 * **kernel %gui and %pylab**: The ``%gui`` and ``%pylab`` magics have been restored
   to the IPython kernel (e.g. in the qtconsole or notebook). This allows activation
   of pylab-mode, or eventloop integration after starting the kernel, which was
   unavailable in 0.11.  Unlike in the terminal, this can be set only once, and
   cannot be changed.
 
 * **%config**: A new ``%config`` magic has been added, giving easy access to the
   IPython configuration system at runtime (:ghpull:`923`).
 
 * **Standalone Kernel**: ``ipython kernel`` subcommand has been added, to allow
   starting a standalone kernel, that can be used with various frontends.
 
 * **Multiline History**: Multiline readline history has been restored to the
   Terminal frontend by default (:ghpull:`838`).
 
 * **%store**: The ``%store`` magic from earlier versions has been updated and
-  placed in an extension, :ref:`extensions_storemagic`. Add 'storemagic' to ``c.InteractiveShellApp.extensions``
-  in ipython_config.py to enable it (:ghpull:`1029`).
+  re-enabled (:ref:`extensions_storemagic`; :ghpull:`1029`). To autorestore
+  stored variables on startup, specify ``c.StoreMagic.autorestore = True`` in
+  :file:`ipython_config.py`.
 
 
 
 Major Bugs fixed
 ----------------
 
 * Simple configuration errors should no longer crash IPython. In 0.11, errors in
   config files, as well as invalid trait values, could crash IPython. Now, such
   errors are reported, and help is displayed.
 
 * Certain SyntaxErrors no longer crash IPython (e.g. just typing keywords, such as
   ``return``, ``break``, etc.). See :ghissue:`704`.
 
 * IPython path utils, such as :func:`~IPython.utils.path.get_ipython_dir` now check
   for write permissions, so IPython should function on systems where the default
   path resolution might point to a read-only location, such as ``HOMESHARE`` on
   Windows (:ghissue:`669`).
 
 * :func:`raw_input` now works in the kernel when multiple frontends are in use. The
   request will be sent to the frontend that made the request, and an exception is
   raised if that frontend does not support stdin requests (e.g. the notebook)
   (:ghissue:`673`).
 
 * :mod:`zmq` version detection no longer uses simple lexicographical comparison to
   check minimum version, which prevents 0.11 from working with pyzmq-2.1.10
   (:ghpull:`758`).
 
 * A bug in PySide < 1.0.7 caused crashes on OSX when tooltips were shown
   (:ghissue:`711`). these tooltips are now disabled on old PySide (:ghpull:`963`).
 
 * IPython no longer crashes when started on recent versions of Python 3 in
   Windows (:ghissue:`737`).
 
 * Instances of classes defined interactively can now be pickled (:ghissue:`29`;
   :ghpull:`648`). Note that pickling saves a reference to the class definition,
   so unpickling the instances will only work where the class has been defined.
 
 .. * use bullet list
 
 Backwards incompatible changes
 ------------------------------
 
 * IPython connection information is no longer specified via ip/port directly,
   rather via json connection files.  These files are stored in the security
   directory, and enable us to turn on HMAC message authentication by default,
   significantly improving the security of kernels.  Various utility functions
   have been added to :mod:`IPython.lib.kernel`, for easier connecting to existing
   kernels.
 
 * :class:`~IPython.zmq.kernelmanager.KernelManager` now has one ip, and several port
   traits, rather than several ip/port pair ``_addr`` traits. This better matches the
   rest of the code, where the ip cannot not be set separately for each channel.
 
 * Custom prompts are now configured using a new class,
   :class:`~IPython.core.prompts.PromptManager`, which has traits for :attr:`in_template`,
   :attr:`in2_template` (the ``...:`` continuation prompt), :attr:`out_template`
   and :attr:`rewrite_template`. This uses Python's string formatting system, so
   you can use ``{time}`` and ``{cwd}``, although we have preserved the abbreviations
   from previous versions, e.g. ``\#`` (prompt number) and ``\w`` (working 
   directory). For the list of available fields, refer to the source of
   :file:`IPython/core/prompts.py`.
 
 * The class inheritance of the Launchers in :mod:`IPython.parallel.apps.launcher`
   used by ipcluster has changed, so that trait names are more consistent across
   batch systems. This may require a few renames in your config files, if you
   customized the command-line args for launching controllers and engines. The
   configurable names have also been changed to be clearer that they point to class
   names, and can now be specified by name only, rather than requiring the full
   import path of each class, e.g.::
 
     IPClusterEngines.engine_launcher = 'IPython.parallel.apps.launcher.MPIExecEngineSetLauncher'
     IPClusterStart.controller_launcher = 'IPython.parallel.apps.launcher.SSHControllerLauncher'
 
   would now be specified as::
 
     IPClusterEngines.engine_launcher_class = 'MPIExec'
     IPClusterStart.controller_launcher_class = 'SSH'
 
   The full path will still work, and is necessary for using custom launchers not in
   IPython's launcher module.
 
 * For embedding a shell, note that the parameter ``user_global_ns`` has been
   replaced by ``user_module``, and expects a module-like object, rather than
   a namespace dict. The ``user_ns`` parameter works the same way as before, and
   calling :func:`~IPython.frontend.terminal.embed.embed` with no arguments still
   works the same way.
 
 .. * use bullet list