Merge pull request #950 from minrk/startup

Add directory for startup files, enabling users to add local configuration more simply without having to know how the config system works much at all.

Now, `profile/startup` dir is checked for files, and they are run prior to explicit exec_code and exec_files.

A short README is added to the dir explaining how they work, and a short paragraph is added to the docs, in both the interactive tutorial and config overview.

IPython/config/profile/README_STARTUP

@@ -0,0 +1,11 b''
+This is the IPython startup directory
+
+.py and .ipy files in this directory will be run *prior* to any code or files specified
+via the exec_lines or exec_files configurables whenever you load this profile.
+
+Files will be run in lexicographical order, so you can control the execution order of files
+with a prefix, e.g.::
+
+    00-first.py
+    50-middle.py
+    99-last.ipy

IPython/core/tests/test_profile.py

@@ -0,0 +1,103 b''
+"""Tests for profile-related functions.
+
+Currently only the startup-dir functionality is tested, but more tests should
+be added for:
+
+    * ipython profile create
+    * ipython profile list
+    * ipython profile create --parallel
+    * security dir permissions
+
+Authors
+-------
+
+* MinRK
+
+"""
+from __future__ import absolute_import
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import os
+import shutil
+import sys
+import tempfile
+
+import nose.tools as nt
+from nose import SkipTest
+
+from IPython.core.profiledir import ProfileDir
+
+from IPython.testing import decorators as dec
+from IPython.testing import tools as tt
+from IPython.utils import py3compat
+
+
+#-----------------------------------------------------------------------------
+# Globals
+#-----------------------------------------------------------------------------
+TMP_TEST_DIR = tempfile.mkdtemp()
+HOME_TEST_DIR = os.path.join(TMP_TEST_DIR, "home_test_dir")
+IP_TEST_DIR = os.path.join(HOME_TEST_DIR,'.ipython')
+
+#
+# Setup/teardown functions/decorators
+#
+
+def setup():
+    """Setup test environment for the module:
+
+            - Adds dummy home dir tree
+    """
+    # Do not mask exceptions here.  In particular, catching WindowsError is a
+    # problem because that exception is only defined on Windows...
+    os.makedirs(IP_TEST_DIR)
+
+
+def teardown():
+    """Teardown test environment for the module:
+
+            - Remove dummy home dir tree
+    """
+    # Note: we remove the parent test dir, which is the root of all test
+    # subdirs we may have created.  Use shutil instead of os.removedirs, so
+    # that non-empty directories are all recursively removed.
+    shutil.rmtree(TMP_TEST_DIR)
+
+
+#-----------------------------------------------------------------------------
+# Test functions
+#-----------------------------------------------------------------------------
+
+def test_startup_py():
+    # create profile dir
+    pd = ProfileDir.create_profile_dir_by_name(IP_TEST_DIR, 'test')
+    # write startup python file
+    with open(os.path.join(pd.startup_dir, '00-start.py'), 'w') as f:
+        f.write('zzz=123\n')
+    # write simple test file, to check that the startup file was run
+    fname = os.path.join(TMP_TEST_DIR, 'test.py')
+    with open(fname, 'w') as f:
+        f.write('print zzz\n')
+    # validate output
+    tt.ipexec_validate(fname, '123', '', 
+        options=['--ipython-dir', IP_TEST_DIR, '--profile', 'test'])
+
+def test_startup_ipy():
+    # create profile dir
+    pd = ProfileDir.create_profile_dir_by_name(IP_TEST_DIR, 'test')
+    # write startup ipython file
+    with open(os.path.join(pd.startup_dir, '00-start.ipy'), 'w') as f:
+        f.write('%profile\n')
+    # write empty script, because we don't need anything to happen
+    # after the startup file is run
+    fname = os.path.join(TMP_TEST_DIR, 'test.py')
+    with open(fname, 'w') as f:
+        f.write('')
+    # validate output
+    tt.ipexec_validate(fname, 'test', '', 
+        options=['--ipython-dir', IP_TEST_DIR, '--profile', 'test'])
+    
+    
  No newline at end of file

IPython/core/profiledir.py

@@ -1,213 +1,228 b''
 # encoding: utf-8
 """
 An object for managing IPython profile directories.
 
 Authors:
 
 * Brian Granger
 * Fernando Perez
 * Min RK
 
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import os
 import shutil
 import sys
 
 from IPython.config.configurable import LoggingConfigurable
 from IPython.config.loader import Config
 from IPython.utils.path import get_ipython_package_dir, expand_path
 from IPython.utils.traitlets import List, Unicode, Bool
 
 #-----------------------------------------------------------------------------
 # Classes and functions
 #-----------------------------------------------------------------------------
 
 
 #-----------------------------------------------------------------------------
 # Module errors
 #-----------------------------------------------------------------------------
 
 class ProfileDirError(Exception):
     pass
 
 
 #-----------------------------------------------------------------------------
 # Class for managing profile directories
 #-----------------------------------------------------------------------------
 
 class ProfileDir(LoggingConfigurable):
     """An object to manage the profile directory and its resources.
 
     The profile directory is used by all IPython applications, to manage
     configuration, logging and security.
 
     This object knows how to find, create and manage these directories. This
     should be used by any code that wants to handle profiles.
     """
 
     security_dir_name = Unicode('security')
     log_dir_name = Unicode('log')
+    startup_dir_name = Unicode('startup')
     pid_dir_name = Unicode('pid')
     security_dir = Unicode(u'')
     log_dir = Unicode(u'')
+    startup_dir = Unicode(u'')
     pid_dir = Unicode(u'')
 
     location = Unicode(u'', config=True,
         help="""Set the profile location directly. This overrides the logic used by the
         `profile` option.""",
         )
 
     _location_isset = Bool(False) # flag for detecting multiply set location
 
     def _location_changed(self, name, old, new):
         if self._location_isset:
             raise RuntimeError("Cannot set profile location more than once.")
         self._location_isset = True
         if not os.path.isdir(new):
             os.makedirs(new)
         
         # ensure config files exist:
         self.security_dir = os.path.join(new, self.security_dir_name)
         self.log_dir = os.path.join(new, self.log_dir_name)
+        self.startup_dir = os.path.join(new, self.startup_dir_name)
         self.pid_dir = os.path.join(new, self.pid_dir_name)
         self.check_dirs()
 
     def _log_dir_changed(self, name, old, new):
         self.check_log_dir()
 
     def check_log_dir(self):
         if not os.path.isdir(self.log_dir):
             os.mkdir(self.log_dir)
 
+    def _startup_dir_changed(self, name, old, new):
+        self.check_startup_dir()
+
+    def check_startup_dir(self):
+        if not os.path.isdir(self.startup_dir):
+            os.mkdir(self.startup_dir)
+        readme = os.path.join(self.startup_dir, 'README')
+        src = os.path.join(get_ipython_package_dir(), u'config', u'profile', u'README_STARTUP')
+        if not os.path.exists(readme):
+            shutil.copy(src, readme)
+
     def _security_dir_changed(self, name, old, new):
         self.check_security_dir()
 
     def check_security_dir(self):
         if not os.path.isdir(self.security_dir):
             os.mkdir(self.security_dir, 0700)
         else:
             try:
                 os.chmod(self.security_dir, 0700)
             except OSError:
                 self.log.warn("Could not set security dir permissions to private.")
 
     def _pid_dir_changed(self, name, old, new):
         self.check_pid_dir()
 
     def check_pid_dir(self):
         if not os.path.isdir(self.pid_dir):
             os.mkdir(self.pid_dir, 0700)
         else:
             try:
                 os.chmod(self.pid_dir, 0700)
             except OSError:
                 self.log.warn("Could not set pid dir permissions to private.")
 
     def check_dirs(self):
         self.check_security_dir()
         self.check_log_dir()
         self.check_pid_dir()
+        self.check_startup_dir()
 
     def copy_config_file(self, config_file, path=None, overwrite=False):
         """Copy a default config file into the active profile directory.
 
         Default configuration files are kept in :mod:`IPython.config.default`.
         This function moves these from that location to the working profile
         directory.
         """
         dst = os.path.join(self.location, config_file)
         if os.path.isfile(dst) and not overwrite:
             return False
         if path is None:
             path = os.path.join(get_ipython_package_dir(), u'config', u'profile', u'default')
         src = os.path.join(path, config_file)
         shutil.copy(src, dst)
         return True
 
     @classmethod
     def create_profile_dir(cls, profile_dir, config=None):
         """Create a new profile directory given a full path.
 
         Parameters
         ----------
         profile_dir : str
             The full path to the profile directory.  If it does exist, it will
             be used.  If not, it will be created.
         """
         return cls(location=profile_dir, config=config)
 
     @classmethod
     def create_profile_dir_by_name(cls, path, name=u'default', config=None):
         """Create a profile dir by profile name and path.
 
         Parameters
         ----------
         path : unicode
             The path (directory) to put the profile directory in.
         name : unicode
             The name of the profile.  The name of the profile directory will
             be "profile_<profile>".
         """
         if not os.path.isdir(path):
             raise ProfileDirError('Directory not found: %s' % path)
         profile_dir = os.path.join(path, u'profile_' + name)
         return cls(location=profile_dir, config=config)
 
     @classmethod
     def find_profile_dir_by_name(cls, ipython_dir, name=u'default', config=None):
         """Find an existing profile dir by profile name, return its ProfileDir.
 
         This searches through a sequence of paths for a profile dir.  If it
         is not found, a :class:`ProfileDirError` exception will be raised.
 
         The search path algorithm is:
         1. ``os.getcwdu()``
         2. ``ipython_dir``
 
         Parameters
         ----------
         ipython_dir : unicode or str
             The IPython directory to use.
         name : unicode or str
             The name of the profile.  The name of the profile directory
             will be "profile_<profile>".
         """
         dirname = u'profile_' + name
         paths = [os.getcwdu(), ipython_dir]
         for p in paths:
             profile_dir = os.path.join(p, dirname)
             if os.path.isdir(profile_dir):
                 return cls(location=profile_dir, config=config)
         else:
             raise ProfileDirError('Profile directory not found in paths: %s' % dirname)
 
     @classmethod
     def find_profile_dir(cls, profile_dir, config=None):
         """Find/create a profile dir and return its ProfileDir.
 
         This will create the profile directory if it doesn't exist.
 
         Parameters
         ----------
         profile_dir : unicode or str
             The path of the profile directory.  This is expanded using
             :func:`IPython.utils.genutils.expand_path`.
         """
         profile_dir = expand_path(profile_dir)
         if not os.path.isdir(profile_dir):
             raise ProfileDirError('Profile directory not found: %s' % profile_dir)
         return cls(location=profile_dir, config=config)
 
 

IPython/core/shellapp.py

@@ -1,268 +1,286 b''
 # encoding: utf-8
 """
 A mixin for :class:`~IPython.core.application.Application` classes that
 launch InteractiveShell instances, load extensions, etc.
 
 Authors
 -------
 
 * Min Ragan-Kelley
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 from __future__ import absolute_import
 
+import glob
 import os
 import sys
 
 from IPython.config.application import boolean_flag
 from IPython.config.configurable import Configurable
 from IPython.config.loader import Config
 from IPython.utils import py3compat
 from IPython.utils.path import filefind
 from IPython.utils.traitlets import Unicode, Instance, List, Bool
 
 #-----------------------------------------------------------------------------
 # Aliases and Flags
 #-----------------------------------------------------------------------------
 
 shell_flags = {}
 
 addflag = lambda *args: shell_flags.update(boolean_flag(*args))
 addflag('autoindent', 'InteractiveShell.autoindent',
         'Turn on autoindenting.', 'Turn off autoindenting.'
 )
 addflag('automagic', 'InteractiveShell.automagic',
         """Turn on the auto calling of magic commands. Type %%magic at the
         IPython  prompt  for  more information.""",
         'Turn off the auto calling of magic commands.'
 )
 addflag('pdb', 'InteractiveShell.pdb',
     "Enable auto calling the pdb debugger after every exception.",
     "Disable auto calling the pdb debugger after every exception."
 )
 # pydb flag doesn't do any config, as core.debugger switches on import,
 # which is before parsing.  This just allows the flag to be passed.
 shell_flags.update(dict(
     pydb = ({},
         """"Use the third party 'pydb' package as debugger, instead of pdb.
         Requires that pydb is installed."""
     )
 ))
 addflag('pprint', 'PlainTextFormatter.pprint',
     "Enable auto pretty printing of results.",
     "Disable auto auto pretty printing of results."
 )
 addflag('color-info', 'InteractiveShell.color_info',
     """IPython can display information about objects via a set of func-
     tions, and optionally can use colors for this, syntax highlighting
     source code and various other elements.  However, because this
     information is passed through a pager (like 'less') and many pagers get
     confused with color codes, this option is off by default.  You can test
     it and turn it on permanently in your ipython_config.py file if it
     works for you.  Test it and turn it on permanently if it works with
     your system.  The magic function %%color_info allows you to toggle this
     interactively for testing.""",
     "Disable using colors for info related things."
 )
 addflag('deep-reload', 'InteractiveShell.deep_reload',
     """Enable deep (recursive) reloading by default. IPython can use the
     deep_reload module which reloads changes in modules recursively (it
     replaces the reload() function, so you don't need to change anything to
     use it). deep_reload() forces a full reload of modules whose code may
     have changed, which the default reload() function does not.  When
     deep_reload is off, IPython will use the normal reload(), but
     deep_reload will still be available as dreload(). This feature is off
     by default [which means that you have both normal reload() and
     dreload()].""",
     "Disable deep (recursive) reloading by default."
 )
 nosep_config = Config()
 nosep_config.InteractiveShell.separate_in = ''
 nosep_config.InteractiveShell.separate_out = ''
 nosep_config.InteractiveShell.separate_out2 = ''
 
 shell_flags['nosep']=(nosep_config, "Eliminate all spacing between prompts.")
 
 
 # it's possible we don't want short aliases for *all* of these:
 shell_aliases = dict(
     autocall='InteractiveShell.autocall',
     colors='InteractiveShell.colors',
     logfile='InteractiveShell.logfile',
     logappend='InteractiveShell.logappend',
     c='InteractiveShellApp.code_to_run',
     ext='InteractiveShellApp.extra_extension',
 )
 shell_aliases['cache-size'] = 'InteractiveShell.cache_size'
 
 #-----------------------------------------------------------------------------
 # Main classes and functions
 #-----------------------------------------------------------------------------
 
 class InteractiveShellApp(Configurable):
     """A Mixin for applications that start InteractiveShell instances.
     
     Provides configurables for loading extensions and executing files
     as part of configuring a Shell environment.
     
     Provides init_extensions() and init_code() methods, to be called
     after init_shell(), which must be implemented by subclasses.
     """
     extensions = List(Unicode, config=True,
         help="A list of dotted module names of IPython extensions to load."
     )
     extra_extension = Unicode('', config=True,
         help="dotted module name of an IPython extension to load."
     )
     def _extra_extension_changed(self, name, old, new):
         if new:
             # add to self.extensions
             self.extensions.append(new)
 
     exec_files = List(Unicode, config=True,
         help="""List of files to run at IPython startup."""
     )
     file_to_run = Unicode('', config=True,
         help="""A file to be run""")
 
     exec_lines = List(Unicode, config=True,
         help="""lines of code to run at IPython startup."""
     )
     code_to_run = Unicode('', config=True,
         help="Execute the given command string."
     )
     pylab_import_all = Bool(True, config=True,
         help="""If true, an 'import *' is done from numpy and pylab,
         when using pylab"""
     )
     shell = Instance('IPython.core.interactiveshell.InteractiveShellABC')
 
     def init_shell(self):
         raise NotImplementedError("Override in subclasses")
     
     def init_extensions(self):
         """Load all IPython extensions in IPythonApp.extensions.
 
         This uses the :meth:`ExtensionManager.load_extensions` to load all
         the extensions listed in ``self.extensions``.
         """
         if not self.extensions:
             return
         try:
             self.log.debug("Loading IPython extensions...")
             extensions = self.extensions
             for ext in extensions:
                 try:
                     self.log.info("Loading IPython extension: %s" % ext)
                     self.shell.extension_manager.load_extension(ext)
                 except:
                     self.log.warn("Error in loading extension: %s" % ext)
                     self.shell.showtraceback()
         except:
             self.log.warn("Unknown error in loading extensions:")
             self.shell.showtraceback()
 
     def init_code(self):
         """run the pre-flight code, specified via exec_lines"""
+        self._run_startup_files()
         self._run_exec_lines()
         self._run_exec_files()
         self._run_cmd_line_code()
 
     def _run_exec_lines(self):
         """Run lines of code in IPythonApp.exec_lines in the user's namespace."""
         if not self.exec_lines:
             return
         try:
             self.log.debug("Running code from IPythonApp.exec_lines...")
             for line in self.exec_lines:
                 try:
                     self.log.info("Running code in user namespace: %s" %
                                   line)
                     self.shell.run_cell(line, store_history=False)
                 except:
                     self.log.warn("Error in executing line in user "
                                   "namespace: %s" % line)
                     self.shell.showtraceback()
         except:
             self.log.warn("Unknown error in handling IPythonApp.exec_lines:")
             self.shell.showtraceback()
 
     def _exec_file(self, fname):
         try:
             full_filename = filefind(fname, [u'.', self.ipython_dir])
         except IOError as e:
             self.log.warn("File not found: %r"%fname)
             return
         # Make sure that the running script gets a proper sys.argv as if it
         # were run from a system shell.
         save_argv = sys.argv
         sys.argv = [full_filename] + self.extra_args[1:]
         # protect sys.argv from potential unicode strings on Python 2:
         if not py3compat.PY3:
             sys.argv = [ py3compat.cast_bytes(a) for a in sys.argv ]
         try:
             if os.path.isfile(full_filename):
                 if full_filename.endswith('.ipy'):
                     self.log.info("Running file in user namespace: %s" %
                                   full_filename)
                     self.shell.safe_execfile_ipy(full_filename)
                 else:
                     # default to python, even without extension
                     self.log.info("Running file in user namespace: %s" %
                                   full_filename)
                     # Ensure that __file__ is always defined to match Python behavior
                     self.shell.user_ns['__file__'] = fname
                     try:
                         self.shell.safe_execfile(full_filename, self.shell.user_ns)
                     finally:
                         del self.shell.user_ns['__file__']
         finally:
             sys.argv = save_argv
 
+    def _run_startup_files(self):
+        """Run files from profile startup directory"""
+        startup_dir = self.profile_dir.startup_dir
+        startup_files = glob.glob(os.path.join(startup_dir, '*.py'))
+        startup_files += glob.glob(os.path.join(startup_dir, '*.ipy'))
+        if not startup_files:
+            return
+        
+        self.log.debug("Running startup files from %s...", startup_dir)
+        try:
+            for fname in sorted(startup_files):
+                self._exec_file(fname)
+        except:
+            self.log.warn("Unknown error in handling startup files:")
+            self.shell.showtraceback()
+
     def _run_exec_files(self):
         """Run files from IPythonApp.exec_files"""
         if not self.exec_files:
             return
 
         self.log.debug("Running files in IPythonApp.exec_files...")
         try:
             for fname in self.exec_files:
                 self._exec_file(fname)
         except:
             self.log.warn("Unknown error in handling IPythonApp.exec_files:")
             self.shell.showtraceback()
 
     def _run_cmd_line_code(self):
         """Run code or file specified at the command-line"""
         if self.code_to_run:
             line = self.code_to_run
             try:
                 self.log.info("Running code given at command line (c=): %s" %
                               line)
                 self.shell.run_cell(line, store_history=False)
             except:
                 self.log.warn("Error in executing line in user namespace: %s" %
                               line)
                 self.shell.showtraceback()
 
         # Like Python itself, ignore the second if the first of these is present
         elif self.file_to_run:
             fname = self.file_to_run
             try:
                 self._exec_file(fname)
             except:
                 self.log.warn("Error in executing file in user namespace: %s" %
                               fname)
                 self.shell.showtraceback()
 

IPython/testing/tools.py

@@ -1,402 +1,402 b''
 """Generic testing tools that do NOT depend on Twisted.
 
 In particular, this module exposes a set of top-level assert* functions that
 can be used in place of nose.tools.assert* in method generators (the ones in
 nose can not, at least as of nose 0.10.4).
 
 Note: our testing package contains testing.util, which does depend on Twisted
 and provides utilities for tests that manage Deferreds.  All testing support
 tools that only depend on nose, IPython or the standard library should go here
 instead.
 
 
 Authors
 -------
 - Fernando Perez <[e-mail]>
 """
 
 from __future__ import absolute_import
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import os
 import re
 import sys
 import tempfile
 
 from contextlib import contextmanager
 from io import StringIO
 
 try:
     # These tools are used by parts of the runtime, so we make the nose
     # dependency optional at this point.  Nose is a hard dependency to run the
     # test suite, but NOT to use ipython itself.
     import nose.tools as nt
     has_nose = True
 except ImportError:
     has_nose = False
 
 from IPython.config.loader import Config
 from IPython.utils.process import find_cmd, getoutputerror
 from IPython.utils.text import list_strings, getdefaultencoding
 from IPython.utils.io import temp_pyfile, Tee
 from IPython.utils import py3compat
 
 from . import decorators as dec
 from . import skipdoctest
 
 #-----------------------------------------------------------------------------
 # Globals
 #-----------------------------------------------------------------------------
 
 # Make a bunch of nose.tools assert wrappers that can be used in test
 # generators.  This will expose an assert* function for each one in nose.tools.
 
 _tpl = """
 def %(name)s(*a,**kw):
     return nt.%(name)s(*a,**kw)
 """
 
 if has_nose:
     for _x in [a for a in dir(nt) if a.startswith('assert')]:
         exec _tpl % dict(name=_x)
 
 #-----------------------------------------------------------------------------
 # Functions and classes
 #-----------------------------------------------------------------------------
 
 # The docstring for full_path doctests differently on win32 (different path
 # separator) so just skip the doctest there.  The example remains informative.
 doctest_deco = skipdoctest.skip_doctest if sys.platform == 'win32' else dec.null_deco
 
 @doctest_deco
 def full_path(startPath,files):
     """Make full paths for all the listed files, based on startPath.
 
     Only the base part of startPath is kept, since this routine is typically
     used with a script's __file__ variable as startPath.  The base of startPath
     is then prepended to all the listed files, forming the output list.
 
     Parameters
     ----------
       startPath : string
         Initial path to use as the base for the results.  This path is split
       using os.path.split() and only its first component is kept.
 
       files : string or list
         One or more files.
 
     Examples
     --------
 
     >>> full_path('/foo/bar.py',['a.txt','b.txt'])
     ['/foo/a.txt', '/foo/b.txt']
 
     >>> full_path('/foo',['a.txt','b.txt'])
     ['/a.txt', '/b.txt']
 
     If a single file is given, the output is still a list:
     >>> full_path('/foo','a.txt')
     ['/a.txt']
     """
 
     files = list_strings(files)
     base = os.path.split(startPath)[0]
     return [ os.path.join(base,f) for f in files ]
 
 
 def parse_test_output(txt):
     """Parse the output of a test run and return errors, failures.
 
     Parameters
     ----------
     txt : str
       Text output of a test run, assumed to contain a line of one of the
       following forms::
         'FAILED (errors=1)'
         'FAILED (failures=1)'
         'FAILED (errors=1, failures=1)'
 
     Returns
     -------
     nerr, nfail: number of errors and failures.
     """
 
     err_m = re.search(r'^FAILED \(errors=(\d+)\)', txt, re.MULTILINE)
     if err_m:
         nerr = int(err_m.group(1))
         nfail = 0
         return  nerr, nfail
 
     fail_m = re.search(r'^FAILED \(failures=(\d+)\)', txt, re.MULTILINE)
     if fail_m:
         nerr = 0
         nfail = int(fail_m.group(1))
         return  nerr, nfail
 
     both_m = re.search(r'^FAILED \(errors=(\d+), failures=(\d+)\)', txt,
                        re.MULTILINE)
     if both_m:
         nerr = int(both_m.group(1))
         nfail = int(both_m.group(2))
         return  nerr, nfail
 
     # If the input didn't match any of these forms, assume no error/failures
     return 0, 0
 
 
 # So nose doesn't think this is a test
 parse_test_output.__test__ = False
 
 
 def default_argv():
     """Return a valid default argv for creating testing instances of ipython"""
 
     return ['--quick', # so no config file is loaded
             # Other defaults to minimize side effects on stdout
             '--colors=NoColor', '--no-term-title','--no-banner',
             '--autocall=0']
 
 
 def default_config():
     """Return a config object with good defaults for testing."""
     config = Config()
     config.TerminalInteractiveShell.colors = 'NoColor'
     config.TerminalTerminalInteractiveShell.term_title = False,
     config.TerminalInteractiveShell.autocall = 0
     config.HistoryManager.hist_file = tempfile.mktemp(u'test_hist.sqlite')
     config.HistoryManager.db_cache_size = 10000
     return config
 
 
 def ipexec(fname, options=None):
     """Utility to call 'ipython filename'.
 
     Starts IPython witha minimal and safe configuration to make startup as fast
     as possible.
 
     Note that this starts IPython in a subprocess!
 
     Parameters
     ----------
     fname : str
       Name of file to be executed (should have .py or .ipy extension).
 
     options : optional, list
       Extra command-line flags to be passed to IPython.
 
     Returns
     -------
     (stdout, stderr) of ipython subprocess.
     """
     if options is None: options = []
 
     # For these subprocess calls, eliminate all prompt printing so we only see
     # output from script execution
     prompt_opts = [ '--InteractiveShell.prompt_in1=""',
                     '--InteractiveShell.prompt_in2=""',
                     '--InteractiveShell.prompt_out=""'
     ]
     cmdargs = ' '.join(default_argv() + prompt_opts + options)
 
     _ip = get_ipython()
     test_dir = os.path.dirname(__file__)
 
     ipython_cmd = find_cmd('ipython3' if py3compat.PY3 else 'ipython')
     # Absolute path for filename
     full_fname = os.path.join(test_dir, fname)
     full_cmd = '%s %s %s' % (ipython_cmd, cmdargs, full_fname)
     #print >> sys.stderr, 'FULL CMD:', full_cmd # dbg
     out = getoutputerror(full_cmd)
     # `import readline` causes 'ESC[?1034h' to be the first output sometimes,
     # so strip that off the front of the first line if it is found
     if out:
         first = out[0]
         m = re.match(r'\x1b\[[^h]+h', first)
         if m:
             # strip initial readline escape
             out = list(out)
             out[0] = first[len(m.group()):]
             out = tuple(out)
     return out
 
 
 def ipexec_validate(fname, expected_out, expected_err='',
                     options=None):
     """Utility to call 'ipython filename' and validate output/error.
 
     This function raises an AssertionError if the validation fails.
 
     Note that this starts IPython in a subprocess!
 
     Parameters
     ----------
     fname : str
       Name of the file to be executed (should have .py or .ipy extension).
 
     expected_out : str
       Expected stdout of the process.
 
     expected_err : optional, str
       Expected stderr of the process.
 
     options : optional, list
       Extra command-line flags to be passed to IPython.
 
     Returns
     -------
     None
     """
 
     import nose.tools as nt
 
-    out, err = ipexec(fname)
+    out, err = ipexec(fname, options)
     #print 'OUT', out  # dbg
     #print 'ERR', err  # dbg
     # If there are any errors, we must check those befor stdout, as they may be
     # more informative than simply having an empty stdout.
     if err:
         if expected_err:
             nt.assert_equals(err.strip(), expected_err.strip())
         else:
             raise ValueError('Running file %r produced error: %r' %
                              (fname, err))
     # If no errors or output on stderr was expected, match stdout
     nt.assert_equals(out.strip(), expected_out.strip())
 
 
 class TempFileMixin(object):
     """Utility class to create temporary Python/IPython files.
 
     Meant as a mixin class for test cases."""
 
     def mktmp(self, src, ext='.py'):
         """Make a valid python temp file."""
         fname, f = temp_pyfile(src, ext)
         self.tmpfile = f
         self.fname = fname
 
     def tearDown(self):
         if hasattr(self, 'tmpfile'):
             # If the tmpfile wasn't made because of skipped tests, like in
             # win32, there's nothing to cleanup.
             self.tmpfile.close()
             try:
                 os.unlink(self.fname)
             except:
                 # On Windows, even though we close the file, we still can't
                 # delete it.  I have no clue why
                 pass
 
 pair_fail_msg = ("Testing {0}\n\n"
                 "In:\n"
                 "  {1!r}\n"
                 "Expected:\n"
                 "  {2!r}\n"
                 "Got:\n"
                 "  {3!r}\n")
 def check_pairs(func, pairs):
     """Utility function for the common case of checking a function with a
     sequence of input/output pairs.
 
     Parameters
     ----------
     func : callable
       The function to be tested. Should accept a single argument.
     pairs : iterable
       A list of (input, expected_output) tuples.
 
     Returns
     -------
     None. Raises an AssertionError if any output does not match the expected
     value.
     """
     name = getattr(func, "func_name", getattr(func, "__name__", "<unknown>"))
     for inp, expected in pairs:
         out = func(inp)
         assert out == expected, pair_fail_msg.format(name, inp, expected, out)
 
 
 if py3compat.PY3:
     MyStringIO = StringIO
 else:
     # In Python 2, stdout/stderr can have either bytes or unicode written to them,
     # so we need a class that can handle both.
     class MyStringIO(StringIO):
         def write(self, s):
             s = py3compat.cast_unicode(s, encoding=getdefaultencoding())
             super(MyStringIO, self).write(s)
 
 notprinted_msg = """Did not find {0!r} in printed output (on {1}):
 {2!r}"""
 
 class AssertPrints(object):
     """Context manager for testing that code prints certain text.
     
     Examples
     --------
     >>> with AssertPrints("abc", suppress=False):
     ...     print "abcd"
     ...     print "def"
     ... 
     abcd
     def
     """
     def __init__(self, s, channel='stdout', suppress=True):
         self.s = s
         self.channel = channel
         self.suppress = suppress
     
     def __enter__(self):
         self.orig_stream = getattr(sys, self.channel)
         self.buffer = MyStringIO()
         self.tee = Tee(self.buffer, channel=self.channel)
         setattr(sys, self.channel, self.buffer if self.suppress else self.tee)
     
     def __exit__(self, etype, value, traceback):
         self.tee.flush()
         setattr(sys, self.channel, self.orig_stream)
         printed = self.buffer.getvalue()
         assert self.s in printed, notprinted_msg.format(self.s, self.channel, printed)
         return False
     
 class AssertNotPrints(AssertPrints):
     """Context manager for checking that certain output *isn't* produced.
     
     Counterpart of AssertPrints"""
     def __exit__(self, etype, value, traceback):
         self.tee.flush()
         setattr(sys, self.channel, self.orig_stream)
         printed = self.buffer.getvalue()
         assert self.s not in printed, notprinted_msg.format(self.s, self.channel, printed)
         return False
 
 @contextmanager
 def mute_warn():
     from IPython.utils import warn
     save_warn = warn.warn
     warn.warn = lambda *a, **kw: None
     try:
         yield
     finally:
         warn.warn = save_warn
 
 @contextmanager
 def make_tempfile(name):
     """ Create an empty, named, temporary file for the duration of the context.
     """
     f = open(name, 'w')
     f.close()
     try:
         yield
     finally:
         os.unlink(name)

docs/source/config/overview.txt

@@ -1,499 +1,528 b''
 .. _config_overview:
 
 ============================================
 Overview of the IPython configuration system
 ============================================
 
 This section describes the IPython configuration system. Starting with version
 0.11, IPython has a completely new configuration system that is quite
 different from the older :file:`ipythonrc` or :file:`ipy_user_conf.py`
 approaches. The new configuration system was designed from scratch to address
 the particular configuration needs of IPython. While there are many
 other excellent configuration systems out there, we found that none of them
 met our requirements.
 
 .. warning::
 
     If you are upgrading to version 0.11 of IPython, you will need to migrate
     your old :file:`ipythonrc` or :file:`ipy_user_conf.py` configuration files
     to the new system.  Read on for information on how to do this.
 
 The discussion that follows is focused on teaching users how to configure
 IPython to their liking.  Developers who want to know more about how they
 can enable their objects to take advantage of the configuration system
 should consult our :ref:`developer guide <developer_guide>`
 
 The main concepts
 =================
 
 There are a number of abstractions that the IPython configuration system uses.
 Each of these abstractions is represented by a Python class.
 
 Configuration object: :class:`~IPython.config.loader.Config`
     A configuration object is a simple dictionary-like class that holds
     configuration attributes and sub-configuration objects. These classes
     support dotted attribute style access (``Foo.bar``) in addition to the
     regular dictionary style access (``Foo['bar']``). Configuration objects
     are smart. They know how to merge themselves with other configuration
     objects and they automatically create sub-configuration objects.
 
 Application: :class:`~IPython.config.application.Application`
     An application is a process that does a specific job. The most obvious
     application is the :command:`ipython` command line program. Each
     application reads *one or more* configuration files and a single set of
     command line options
     and then produces a master configuration object for the application. This
     configuration object is then passed to the configurable objects that the
     application creates. These configurable objects implement the actual logic
     of the application and know how to configure themselves given the
     configuration object.
     
     Applications always have a `log` attribute that is a configured Logger.
     This allows centralized logging configuration per-application.
 
 Configurable: :class:`~IPython.config.configurable.Configurable`
     A configurable is a regular Python class that serves as a base class for
     all main classes in an application. The
     :class:`~IPython.config.configurable.Configurable` base class is
     lightweight and only does one things.
 
     This :class:`~IPython.config.configurable.Configurable` is a subclass
     of :class:`~IPython.utils.traitlets.HasTraits` that knows how to configure
     itself. Class level traits with the metadata ``config=True`` become
     values that can be configured from the command line and configuration
     files.
     
     Developers create :class:`~IPython.config.configurable.Configurable`
     subclasses that implement all of the logic in the application. Each of
     these subclasses has its own configuration information that controls how
     instances are created.
 
 Singletons: :class:`~IPython.config.configurable.SingletonConfigurable`
     Any object for which there is a single canonical instance. These are
     just like Configurables, except they have a class method 
     :meth:`~IPython.config.configurable.SingletonConfigurable.instance`,
     that returns the current active instance (or creates one if it
     does not exist).  Examples of singletons include
     :class:`~IPython.config.application.Application`s and
     :class:`~IPython.core.interactiveshell.InteractiveShell`.  This lets
     objects easily connect to the current running Application without passing
     objects around everywhere.  For instance, to get the current running 
     Application instance, simply do: ``app = Application.instance()``.
 
 
 .. note::
 
     Singletons are not strictly enforced - you can have many instances
     of a given singleton class, but the :meth:`instance` method will always
     return the same one.
 
 Having described these main concepts, we can now state the main idea in our
 configuration system: *"configuration" allows the default values of class
 attributes to be controlled on a class by class basis*. Thus all instances of
 a given class are configured in the same way. Furthermore, if two instances
 need to be configured differently, they need to be instances of two different
 classes. While this model may seem a bit restrictive, we have found that it
 expresses most things that need to be configured extremely well. However, it
 is possible to create two instances of the same class that have different
 trait values. This is done by overriding the configuration.
 
 Now, we show what our configuration objects and files look like.
 
 Configuration objects and files
 ===============================
 
 A configuration file is simply a pure Python file that sets the attributes
 of a global, pre-created configuration object.  This configuration object is a 
 :class:`~IPython.config.loader.Config` instance.  While in a configuration
 file, to get a reference to this object, simply call the :func:`get_config`
 function.  We inject this function into the global namespace that the 
 configuration file is executed in.
 
 Here is an example of a super simple configuration file that does nothing::
 
     c = get_config()
 
 Once you get a reference to the configuration object, you simply set
 attributes on it.  All you have to know is:
 
 * The name of each attribute.
 * The type of each attribute.
 
 The answers to these two questions are provided by the various
 :class:`~IPython.config.configurable.Configurable` subclasses that an
 application uses. Let's look at how this would work for a simple configurable
 subclass::
 
     # Sample configurable:
     from IPython.config.configurable import Configurable
     from IPython.utils.traitlets import Int, Float, Unicode, Bool
     
     class MyClass(Configurable):
         name = Unicode(u'defaultname', config=True)
         ranking = Int(0, config=True)
         value = Float(99.0)
         # The rest of the class implementation would go here..
 
 In this example, we see that :class:`MyClass` has three attributes, two
 of whom (``name``, ``ranking``) can be configured.  All of the attributes
 are given types and default values.  If a :class:`MyClass` is instantiated,
 but not configured, these default values will be used.  But let's see how
 to configure this class in a configuration file::
 
     # Sample config file
     c = get_config()
     
     c.MyClass.name = 'coolname'
     c.MyClass.ranking = 10
 
 After this configuration file is loaded, the values set in it will override
 the class defaults anytime a :class:`MyClass` is created.  Furthermore,
 these attributes will be type checked and validated anytime they are set.
 This type checking is handled by the :mod:`IPython.utils.traitlets` module,
 which provides the :class:`Unicode`, :class:`Int` and :class:`Float` types.
 In addition to these traitlets, the :mod:`IPython.utils.traitlets` provides
 traitlets for a number of other types.
 
 .. note::
 
     Underneath the hood, the :class:`Configurable` base class is a subclass of
     :class:`IPython.utils.traitlets.HasTraits`. The
     :mod:`IPython.utils.traitlets` module is a lightweight version of
     :mod:`enthought.traits`. Our implementation is a pure Python subset
     (mostly API compatible) of :mod:`enthought.traits` that does not have any
     of the automatic GUI generation capabilities. Our plan is to achieve 100%
     API compatibility to enable the actual :mod:`enthought.traits` to
     eventually be used instead. Currently, we cannot use
     :mod:`enthought.traits` as we are committed to the core of IPython being
     pure Python.
 
 It should be very clear at this point what the naming convention is for 
 configuration attributes::
 
     c.ClassName.attribute_name = attribute_value
 
 Here, ``ClassName`` is the name of the class whose configuration attribute you
 want to set, ``attribute_name`` is the name of the attribute you want to set
 and ``attribute_value`` the the value you want it to have. The ``ClassName``
 attribute of ``c`` is not the actual class, but instead is another
 :class:`~IPython.config.loader.Config` instance.
 
 .. note::
 
     The careful reader may wonder how the ``ClassName`` (``MyClass`` in
     the above example) attribute of the configuration object ``c`` gets
     created. These attributes are created on the fly by the
     :class:`~IPython.config.loader.Config` instance, using a simple naming
     convention. Any attribute of a :class:`~IPython.config.loader.Config`
     instance whose name begins with an uppercase character is assumed to be a
     sub-configuration and a new empty :class:`~IPython.config.loader.Config`
     instance is dynamically created for that attribute. This allows deeply
     hierarchical information created easily (``c.Foo.Bar.value``) on the fly.
 
 Configuration files inheritance
 ===============================
 
 Let's say you want to have different configuration files for various purposes.
 Our configuration system makes it easy for one configuration file to inherit
 the information in another configuration file. The :func:`load_subconfig`
 command can be used in a configuration file for this purpose. Here is a simple
 example that loads all of the values from the file :file:`base_config.py`::
 
     # base_config.py
     c = get_config()
     c.MyClass.name = 'coolname'
     c.MyClass.ranking = 100
 
 into the configuration file :file:`main_config.py`::
 
     # main_config.py
     c = get_config()
     
     # Load everything from base_config.py
     load_subconfig('base_config.py')
     
     # Now override one of the values
     c.MyClass.name = 'bettername'
 
 In a situation like this the :func:`load_subconfig` makes sure that the
 search path for sub-configuration files is inherited from that of the parent.
 Thus, you can typically put the two in the same directory and everything will
 just work.
 
 You can also load configuration files by profile, for instance:
 
 .. sourcecode:: python
 
     load_subconfig('ipython_config.py', profile='default')
 
 to inherit your default configuration as a starting point.
 
 
 Class based configuration inheritance
 =====================================
 
 There is another aspect of configuration where inheritance comes into play.
 Sometimes, your classes will have an inheritance hierarchy that you want
 to be reflected in the configuration system.  Here is a simple example::
 
     from IPython.config.configurable import Configurable
     from IPython.utils.traitlets import Int, Float, Unicode, Bool
     
     class Foo(Configurable):
         name = Unicode(u'fooname', config=True)
         value = Float(100.0, config=True)
     
     class Bar(Foo):
         name = Unicode(u'barname', config=True)
         othervalue = Int(0, config=True)
 
 Now, we can create a configuration file to configure instances of :class:`Foo`
 and :class:`Bar`::
 
     # config file
     c = get_config()
     
     c.Foo.name = u'bestname'
     c.Bar.othervalue = 10
 
 This class hierarchy and configuration file accomplishes the following:
 
 * The default value for :attr:`Foo.name` and :attr:`Bar.name` will be
   'bestname'.  Because :class:`Bar` is a :class:`Foo` subclass it also
   picks up the configuration information for :class:`Foo`.
 * The default value for :attr:`Foo.value` and :attr:`Bar.value` will be
   ``100.0``, which is the value specified as the class default.
 * The default value for :attr:`Bar.othervalue` will be 10 as set in the
   configuration file.  Because :class:`Foo` is the parent of :class:`Bar`
   it doesn't know anything about the :attr:`othervalue` attribute.
 
 
 .. _ipython_dir:
 
 Configuration file location
 ===========================
 
 So where should you put your configuration files? IPython uses "profiles" for
 configuration, and by default, all profiles will be stored in the so called
 "IPython directory". The location of this directory is determined by the
 following algorithm:
 
 * If the ``ipython_dir`` command line flag is given, its value is used.
 
 * If not, the value returned by :func:`IPython.utils.path.get_ipython_dir`
   is used. This function will first look at the :envvar:`IPYTHON_DIR`
   environment variable and then default to a platform-specific default.
 
 On posix systems (Linux, Unix, etc.), IPython respects the ``$XDG_CONFIG_HOME``
 part of the `XDG Base Directory`_ specification. If ``$XDG_CONFIG_HOME`` is
 defined and exists ( ``XDG_CONFIG_HOME`` has a default interpretation of
 :file:`$HOME/.config`), then IPython's config directory will be located in
 :file:`$XDG_CONFIG_HOME/ipython`.  If users still have an IPython directory
 in :file:`$HOME/.ipython`, then that will be used. in preference to the 
 system default.
 
 For most users, the default value will simply be something like 
 :file:`$HOME/.config/ipython` on Linux, or :file:`$HOME/.ipython`
 elsewhere.
 
 Once the location of the IPython directory has been determined, you need to know
 which profile you are using. For users with a single configuration, this will
 simply be 'default', and will be located in 
 :file:`<IPYTHON_DIR>/profile_default`.
 
 The next thing you need to know is what to call your configuration file. The
 basic idea is that each application has its own default configuration filename.
 The default named used by the :command:`ipython` command line program is
 :file:`ipython_config.py`, and *all* IPython applications will use this file.
 Other applications, such as the parallel :command:`ipcluster` scripts or the
 QtConsole will load their own config files *after* :file:`ipython_config.py`. To
 load a particular configuration file instead of the default, the name can be
 overridden by the ``config_file`` command line flag.
 
 To generate the default configuration files, do::
 
     $> ipython profile create
 
 and you will have a default :file:`ipython_config.py` in your IPython directory
 under :file:`profile_default`. If you want the default config files for the
 :mod:`IPython.parallel` applications, add ``--parallel`` to the end of the
 command-line args.
 
 .. _Profiles:
 
 Profiles
 ========
 
 A profile is a directory containing configuration and runtime files, such as
 logs, connection info for the parallel apps, and your IPython command history.
 
 The idea is that users often want to maintain a set of configuration files for
 different purposes: one for doing numerical computing with NumPy and SciPy and
 another for doing symbolic computing with SymPy. Profiles make it easy to keep a
 separate configuration files, logs, and histories for each of these purposes.
 
 Let's start by showing how a profile is used:
 
 .. code-block:: bash
 
     $ ipython --profile=sympy
 
 This tells the :command:`ipython` command line program to get its configuration
 from the "sympy" profile. The file names for various profiles do not change. The
 only difference is that profiles are named in a special way. In the case above,
 the "sympy" profile means looking for :file:`ipython_config.py` in :file:`<IPYTHON_DIR>/profile_sympy`.
 
 The general pattern is this: simply create a new profile with:
 
 .. code-block:: bash
 
     ipython profile create <name>
 
 which adds a directory called ``profile_<name>`` to your IPython directory. Then
 you can load this profile by adding ``--profile=<name>`` to your command line
 options. Profiles are supported by all IPython applications.
 
 IPython ships with some sample profiles in :file:`IPython/config/profile`. If
 you create profiles with the name of one of our shipped profiles, these config
 files will be copied over instead of starting with the automatically generated
 config files.
 
+Security Files
+--------------
+
+If you are using the notebook, qtconsole, or parallel code, IPython stores
+connection information in small JSON files in the active profile's security
+directory. This directory is made private, so only you can see the files inside. If
+you need to move connection files around to other computers, this is where they will
+be. If you want your code to be able to open security files by name, we have a
+convenience function :func:`IPython.utils.path.get_security_file`, which will return
+the absolute path to a security file from its filename and [optionally] profile
+name.
+
+Startup Files
+-------------
+
+If you want some code to be run at the beginning of every IPython session with a
+particular profile, the easiest way is to add Python (.py) or IPython (.ipy) scripts
+to your :file:`<profile>/startup` directory. Files in this directory will always be
+executed as soon as the IPython shell is constructed, and before any other code or
+scripts you have specified. If you have multiple files in the startup directory,
+they will be run in lexicographical order, so you can control the ordering by adding
+a '00-' prefix.
+
+.. note::
+
+    Automatic startup files are new in IPython 0.12. Use the
+    InteractiveShellApp.exec_files configurable for similar behavior in 0.11.
+
+
 .. _commandline:
 
 Command-line arguments
 ======================
 
 IPython exposes *all* configurable options on the command-line. The command-line
 arguments are generated from the Configurable traits of the classes associated
 with a given Application.  Configuring IPython from the command-line may look
 very similar to an IPython config file
 
 IPython applications use a parser called
 :class:`~IPython.config.loader.KeyValueLoader` to load values into a Config
 object.  Values are assigned in much the same way as in a config file:
 
 .. code-block:: bash
 
     $> ipython --InteractiveShell.use_readline=False --BaseIPythonApplication.profile='myprofile'
 
 Is the same as adding:
 
 .. sourcecode:: python
 
     c.InteractiveShell.use_readline=False
     c.BaseIPythonApplication.profile='myprofile'
 
 to your config file. Key/Value arguments *always* take a value, separated by '='
 and no spaces.
 
 Common Arguments
 ****************
 
 Since the strictness and verbosity of the KVLoader above are not ideal for everyday
 use, common arguments can be specified as flags_ or aliases_.
 
 Flags and Aliases are handled by :mod:`argparse` instead, allowing for more flexible
 parsing. In general, flags and aliases are prefixed by ``--``, except for those
 that are single characters, in which case they can be specified with a single ``-``, e.g.:
 
 .. code-block:: bash
 
     $> ipython -i -c "import numpy; x=numpy.linspace(0,1)" --profile testing --colors=lightbg
 
 Aliases
 -------
 
 For convenience, applications have a mapping of commonly used traits, so you don't have
 to specify the whole class name:
 
 .. code-block:: bash
 
     $> ipython --profile myprofile
     # and
     $> ipython --profile='myprofile'
     # are equivalent to
     $> ipython --BaseIPythonApplication.profile='myprofile'
 
 Flags
 -----
 
 Applications can also be passed **flags**. Flags are options that take no
 arguments. They are simply wrappers for
 setting one or more configurables with predefined values, often True/False.
 
 For instance:
 
 .. code-block:: bash
 
     $> ipcontroller --debug
     # is equivalent to
     $> ipcontroller --Application.log_level=DEBUG
     # and
     $> ipython --pylab
     # is equivalent to
     $> ipython --pylab=auto
     # or
     $> ipython --no-banner
     # is equivalent to
     $> ipython --TerminalIPythonApp.display_banner=False
 
 Subcommands
 ***********
 
 
 Some IPython applications have **subcommands**. Subcommands are modeled after
 :command:`git`, and are called with the form :command:`command subcommand
 [...args]`.  Currently, the QtConsole is a subcommand of terminal IPython:
 
 .. code-block:: bash
 
     $> ipython qtconsole --profile=myprofile
 
 and :command:`ipcluster` is simply a wrapper for its various subcommands (start,
 stop, engines).
 
 .. code-block:: bash
 
     $> ipcluster start --profile=myprofile --n=4
 
 
 To see a list of the available aliases, flags, and subcommands for an IPython application, simply pass ``-h`` or ``--help``.  And to see the full list of configurable options (*very* long), pass ``--help-all``.
 
 
 Design requirements
 ===================
 
 Here are the main requirements we wanted our configuration system to have:
 
 * Support for hierarchical configuration information.
 
 * Full integration with command line option parsers.  Often, you want to read
   a configuration file, but then override some of the values with command line
   options.  Our configuration system automates this process and allows each
   command line option to be linked to a particular attribute in the 
   configuration hierarchy that it will override.
 
 * Configuration files that are themselves valid Python code. This accomplishes
   many things. First, it becomes possible to put logic in your configuration
   files that sets attributes based on your operating system, network setup,
   Python version, etc. Second, Python has a super simple syntax for accessing
   hierarchical data structures, namely regular attribute access
   (``Foo.Bar.Bam.name``). Third, using Python makes it easy for users to
   import configuration attributes from one configuration file to another.
   Fourth, even though Python is dynamically typed, it does have types that can
   be checked at runtime. Thus, a ``1`` in a config file is the integer '1',
   while a ``'1'`` is a string.
 
 * A fully automated method for getting the configuration information to the
   classes that need it at runtime. Writing code that walks a configuration
   hierarchy to extract a particular attribute is painful. When you have
   complex configuration information with hundreds of attributes, this makes
   you want to cry.
 
 * Type checking and validation that doesn't require the entire configuration
   hierarchy to be specified statically before runtime. Python is a very
   dynamic language and you don't always know everything that needs to be
   configured when a program starts.
 
 
 .. _`XDG Base Directory`: http://standards.freedesktop.org/basedir-spec/basedir-spec-latest.html

docs/source/interactive/tutorial.txt

@@ -1,145 +1,160 b''
 .. _tutorial:
 
 ======================
 Introducing IPython
 ======================
 
 You don't need to know anything beyond Python to start using IPython – just type
 commands as you would at the standard Python prompt. But IPython can do much
 more than the standard prompt. Some key features are described here. For more
 information, check the :ref:`tips page <tips>`, or look at examples in the
 `IPython cookbook <http://wiki.ipython.org/index.php?title=Cookbook>`_.
 
 If you've never used Python before, you might want to look at `the official
 tutorial <http://docs.python.org/tutorial/>`_ or an alternative, `Dive into
 Python <http://diveintopython.org/toc/index.html>`_.
 
 Tab completion
 ==============
 
 Tab completion, especially for attributes, is a convenient way to explore the
 structure of any object you're dealing with. Simply type ``object_name.<TAB>``
 to view the object's attributes (see :ref:`the readline section <readline>` for
 more). Besides Python objects and keywords, tab completion also works on file
 and directory names.
 
 Exploring your objects
 ======================
 
 Typing ``object_name?`` will print all sorts of details about any object,
 including docstrings, function definition lines (for call arguments) and
 constructor details for classes. To get specific information on an object, you
 can use the magic commands ``%pdoc``, ``%pdef``, ``%psource`` and ``%pfile``
 
 Magic functions
 ===============
 
 IPython has a set of predefined 'magic functions' that you can call with a
 command line style syntax. These include:
 
 - Functions that work with code: ``%run``, ``%edit``, ``%save``, ``%macro``,
   ``%recall``, etc.
 - Functions which affect the shell: ``%colors``, ``%xmode``, ``%autoindent``, etc.
 - Other functions such as ``%reset``, ``%timeit`` or ``%paste``.
 
 You can always call these using the % prefix, and if you're typing one on a line
 by itself, you can omit even that::
 
     run thescript.py
     
 For more details on any magic function, call ``%somemagic?`` to read its
 docstring. To see all the available magic functions, call ``%lsmagic``.
 
 Running and Editing
 -------------------
 
 The %run magic command allows you to run any python script and load all of its
 data directly into the interactive namespace. Since the file is re-read from
 disk each time, changes you make to it are reflected immediately (unlike
 imported modules, which have to be specifically reloaded). IPython also includes
 :ref:`dreload <dreload>`, a recursive reload function.
 
 %run has special flags for timing the execution of your scripts (-t), or for
 running them under the control of either Python's pdb debugger (-d) or
 profiler (-p).
 
 The %edit command gives a reasonable approximation of multiline editing,
 by invoking your favorite editor on the spot. IPython will execute the
 code you type in there as if it were typed interactively.
 
 Debugging
 ---------
 
 After an exception occurs, you can call ``%debug`` to jump into the Python
 debugger (pdb) and examine the problem. Alternatively, if you call ``%pdb``,
 IPython will automatically start the debugger on any uncaught exception. You can
 print variables, see code, execute statements and even walk up and down the
 call stack to track down the true source of the problem. Running programs with
 %run and pdb active can be an efficient way to develop and debug code, in many
 cases eliminating the need for print statements or external debugging tools.
 
 You can also step through a program from the beginning by calling
 ``%run -d theprogram.py``.
 
 History
 =======
 
 IPython stores both the commands you enter, and the results it produces. You
 can easily go through previous commands with the up- and down-arrow keys, or
 access your history in more sophisticated ways.
 
 Input and output history are kept in variables called ``In`` and ``Out``, which
 can both be indexed by the prompt number on which they occurred, e.g. ``In[4]``.
 The last three objects in output history are also kept in variables named ``_``,
 ``__`` and ``___``.
 
 You can use the ``%history`` magic function to examine past input and output.
 Input history from previous sessions is saved in a database, and IPython can be
 configured to save output history.
 
 Several other magic functions can use your input history, including ``%edit``, 
 ``%rerun``, ``%recall``, ``%macro``, ``%save`` and ``%pastebin``. You can use a
 standard format to refer to lines::
 
     %pastebin 3 18-20 ~1/1-5
     
 This will take line 3 and lines 18 to 20 from the current session, and lines
 1-5 from the previous session.
 
 System shell commands
 =====================
 
 To run any command at the system shell, simply prefix it with !, e.g.::
 
     !ping www.bbc.co.uk
     
 You can capture the output into a Python list, e.g.: ``files = !ls``. To pass
 the values of Python variables or expressions to system commands, prefix them
 with $: ``!grep -rF $pattern ipython/*``. See :ref:`our shell section
 <system_shell_access>` for more details.
 	
 Define your own system aliases
 ------------------------------
 
 It's convenient to have aliases to the system commands you use most often.
 This allows you to work seamlessly from inside IPython with the same commands
 you are used to in your system shell. IPython comes with some pre-defined
 aliases and a complete system for changing directories, both via a stack (see
 %pushd, %popd and %dhist) and via direct %cd. The latter keeps a history of
 visited directories and allows you to go to any previously visited one.
 
 
 Configuration
 =============
 
 Much of IPython can be tweaked through configuration. To get started, use the
 command ``ipython profile create`` to produce the default config files. These
 will be placed in :file:`~/.ipython/profile_default` or
 :file:`~/.config/ipython/profile_default`, and contain comments explaining what
 the various options do.
 
 Profiles allow you to use IPython for different tasks, keeping separate config
 files and history for each one. More details in :ref:`the profiles section
 <profiles>`.
 
+Startup Files
+-------------
+
+If you want some code to be run at the beginning of every IPython session, the
+easiest way is to add Python (.py) or IPython (.ipy) scripts to your
+:file:`<profile>/startup` directory. Files in this directory will always be executed
+as soon as the IPython shell is constructed, and before any other code or scripts
+you have specified. If you have multiple files in the startup directory, they will
+be run in lexicographical order, so you can control the ordering by adding a '00-'
+prefix.
+
+.. note::
+
+    Automatic startup files are new in IPython 0.12. Use the
+    InteractiveShellApp.exec_files configurable for similar behavior in 0.11.
 

setupbase.py

@@ -1,382 +1,382 b''
 # encoding: utf-8
 """
 This module defines the things that are used in setup.py for building IPython
 
 This includes:
 
     * The basic arguments to setup
     * Functions for finding things like packages, package data, etc.
     * A function for checking dependencies.
 """
 from __future__ import print_function
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
 import os
 import sys
 
 try:
     from configparser import ConfigParser
 except:
     from ConfigParser import ConfigParser
 from distutils.command.build_py import build_py
 from glob import glob
 
 from setupext import install_data_ext
 
 #-------------------------------------------------------------------------------
 # Useful globals and utility functions
 #-------------------------------------------------------------------------------
 
 # A few handy globals
 isfile = os.path.isfile
 pjoin = os.path.join
 
 def oscmd(s):
     print(">", s)
     os.system(s)
 
 try:
     execfile
 except NameError:
     def execfile(fname, globs, locs=None):
         locs = locs or globs
         exec(compile(open(fname).read(), fname, "exec"), globs, locs)
 
 # A little utility we'll need below, since glob() does NOT allow you to do
 # exclusion on multiple endings!
 def file_doesnt_endwith(test,endings):
     """Return true if test is a file and its name does NOT end with any
     of the strings listed in endings."""
     if not isfile(test):
         return False
     for e in endings:
         if test.endswith(e):
             return False
     return True
 
 #---------------------------------------------------------------------------
 # Basic project information
 #---------------------------------------------------------------------------
 
 # release.py contains version, authors, license, url, keywords, etc.
 execfile(pjoin('IPython','core','release.py'), globals())
 
 # Create a dict with the basic information
 # This dict is eventually passed to setup after additional keys are added.
 setup_args = dict(
       name             = name,
       version          = version,
       description      = description,
       long_description = long_description,
       author           = author,
       author_email     = author_email,
       url              = url,
       download_url     = download_url,
       license          = license,
       platforms        = platforms,
       keywords         = keywords,
       classifiers      = classifiers,
       cmdclass         = {'install_data': install_data_ext},
       )
 
 
 #---------------------------------------------------------------------------
 # Find packages
 #---------------------------------------------------------------------------
 
 def find_packages():
     """
     Find all of IPython's packages.
     """
     excludes = ['deathrow']
     packages = []
     for dir,subdirs,files in os.walk('IPython'):
         package = dir.replace(os.path.sep, '.')
         if any([ package.startswith('IPython.'+exc) for exc in excludes ]):
             # package is to be excluded (e.g. deathrow)
             continue
         if '__init__.py' not in files:
             # not a package
             continue
         packages.append(package)
     return packages
 
 #---------------------------------------------------------------------------
 # Find package data
 #---------------------------------------------------------------------------
 
 def find_package_data():
     """
     Find IPython's package_data.
     """
     # This is not enough for these things to appear in an sdist.
     # We need to muck with the MANIFEST to get this to work
 
     # walk notebook resources:
     cwd = os.getcwd()
     os.chdir(os.path.join('IPython', 'frontend', 'html', 'notebook'))
     static_walk = list(os.walk('static'))
     os.chdir(cwd)
     static_data = []
     for parent, dirs, files in static_walk:
         for f in files:
             static_data.append(os.path.join(parent, f))
 
     package_data = {
-        'IPython.config.profile' : ['README', '*/*.py'],
+        'IPython.config.profile' : ['README*', '*/*.py'],
         'IPython.testing' : ['*.txt'],
         'IPython.frontend.html.notebook' : ['templates/*'] + static_data,
         'IPython.frontend.qt.console' : ['resources/icon/*.svg'],
     }
     return package_data
 
 
 #---------------------------------------------------------------------------
 # Find data files
 #---------------------------------------------------------------------------
 
 def make_dir_struct(tag,base,out_base):
     """Make the directory structure of all files below a starting dir.
 
     This is just a convenience routine to help build a nested directory
     hierarchy because distutils is too stupid to do this by itself.
 
     XXX - this needs a proper docstring!
     """
 
     # we'll use these a lot below
     lbase = len(base)
     pathsep = os.path.sep
     lpathsep = len(pathsep)
 
     out = []
     for (dirpath,dirnames,filenames) in os.walk(base):
         # we need to strip out the dirpath from the base to map it to the
         # output (installation) path.  This requires possibly stripping the
         # path separator, because otherwise pjoin will not work correctly
         # (pjoin('foo/','/bar') returns '/bar').
 
         dp_eff = dirpath[lbase:]
         if dp_eff.startswith(pathsep):
             dp_eff = dp_eff[lpathsep:]
         # The output path must be anchored at the out_base marker
         out_path = pjoin(out_base,dp_eff)
         # Now we can generate the final filenames. Since os.walk only produces
         # filenames, we must join back with the dirpath to get full valid file
         # paths:
         pfiles = [pjoin(dirpath,f) for f in filenames]
         # Finally, generate the entry we need, which is a pari of (output
         # path, files) for use as a data_files parameter in install_data.
         out.append((out_path, pfiles))
 
     return out
 
 
 def find_data_files():
     """
     Find IPython's data_files.
 
     Most of these are docs.
     """
 
     docdirbase  = pjoin('share', 'doc', 'ipython')
     manpagebase = pjoin('share', 'man', 'man1')
 
     # Simple file lists can be made by hand
     manpages  = filter(isfile, glob(pjoin('docs','man','*.1.gz')))
     if not manpages:
         # When running from a source tree, the manpages aren't gzipped
         manpages = filter(isfile, glob(pjoin('docs','man','*.1')))
     igridhelpfiles = filter(isfile,
                             glob(pjoin('IPython','extensions','igrid_help.*')))
 
     # For nested structures, use the utility above
     example_files = make_dir_struct(
         'data',
         pjoin('docs','examples'),
         pjoin(docdirbase,'examples')
     )
     manual_files = make_dir_struct(
         'data',
         pjoin('docs','html'),
         pjoin(docdirbase,'manual')
     )
 
     # And assemble the entire output list
     data_files = [ (manpagebase, manpages),
                    (pjoin(docdirbase, 'extensions'), igridhelpfiles),
                    ] + manual_files + example_files
 
     return data_files
 
 
 def make_man_update_target(manpage):
     """Return a target_update-compliant tuple for the given manpage.
 
     Parameters
     ----------
     manpage : string
       Name of the manpage, must include the section number (trailing number).
 
     Example
     -------
 
     >>> make_man_update_target('ipython.1') #doctest: +NORMALIZE_WHITESPACE
     ('docs/man/ipython.1.gz',
      ['docs/man/ipython.1'],
      'cd docs/man && gzip -9c ipython.1 > ipython.1.gz')
     """
     man_dir = pjoin('docs', 'man')
     manpage_gz = manpage + '.gz'
     manpath = pjoin(man_dir, manpage)
     manpath_gz = pjoin(man_dir, manpage_gz)
     gz_cmd = ( "cd %(man_dir)s && gzip -9c %(manpage)s > %(manpage_gz)s" %
                locals() )
     return (manpath_gz, [manpath], gz_cmd)
 
 #---------------------------------------------------------------------------
 # Find scripts
 #---------------------------------------------------------------------------
 
 def find_scripts(entry_points=False, suffix=''):
     """Find IPython's scripts.
 
     if entry_points is True:
         return setuptools entry_point-style definitions
     else:
         return file paths of plain scripts [default]
 
     suffix is appended to script names if entry_points is True, so that the
     Python 3 scripts get named "ipython3" etc.
     """
     if entry_points:
         console_scripts = [s % suffix for s in [
             'ipython%s = IPython.frontend.terminal.ipapp:launch_new_instance',
             'pycolor%s = IPython.utils.PyColorize:main',
             'ipcontroller%s = IPython.parallel.apps.ipcontrollerapp:launch_new_instance',
             'ipengine%s = IPython.parallel.apps.ipengineapp:launch_new_instance',
             'iplogger%s = IPython.parallel.apps.iploggerapp:launch_new_instance',
             'ipcluster%s = IPython.parallel.apps.ipclusterapp:launch_new_instance',
             'iptest%s = IPython.testing.iptest:main',
             'irunner%s = IPython.lib.irunner:main'
         ]]
         gui_scripts = [s % suffix for s in [
             'ipython%s-qtconsole = IPython.frontend.qt.console.qtconsoleapp:main',
         ]]
         scripts = dict(console_scripts=console_scripts, gui_scripts=gui_scripts)
     else:
         parallel_scripts = pjoin('IPython','parallel','scripts')
         main_scripts = pjoin('IPython','scripts')
         scripts = [
                    pjoin(parallel_scripts, 'ipengine'),
                    pjoin(parallel_scripts, 'ipcontroller'),
                    pjoin(parallel_scripts, 'ipcluster'),
                    pjoin(parallel_scripts, 'iplogger'),
                    pjoin(main_scripts, 'ipython'),
                    pjoin(main_scripts, 'pycolor'),
                    pjoin(main_scripts, 'irunner'),
                    pjoin(main_scripts, 'iptest')
         ]
     return scripts
 
 #---------------------------------------------------------------------------
 # Verify all dependencies
 #---------------------------------------------------------------------------
 
 def check_for_dependencies():
     """Check for IPython's dependencies.
 
     This function should NOT be called if running under setuptools!
     """
     from setupext.setupext import (
         print_line, print_raw, print_status,
         check_for_sphinx, check_for_pygments,
         check_for_nose, check_for_pexpect,
         check_for_pyzmq, check_for_readline
     )
     print_line()
     print_raw("BUILDING IPYTHON")
     print_status('python', sys.version)
     print_status('platform', sys.platform)
     if sys.platform == 'win32':
         print_status('Windows version', sys.getwindowsversion())
 
     print_raw("")
     print_raw("OPTIONAL DEPENDENCIES")
 
     check_for_sphinx()
     check_for_pygments()
     check_for_nose()
     check_for_pexpect()
     check_for_pyzmq()
     check_for_readline()
 
 def record_commit_info(pkg_dir, build_cmd=build_py):
     """ Return extended build command class for recording commit
 
     The extended command tries to run git to find the current commit, getting
     the empty string if it fails.  It then writes the commit hash into a file
     in the `pkg_dir` path, named ``.git_commit_info.ini``.
 
     In due course this information can be used by the package after it is
     installed, to tell you what commit it was installed from if known.
 
     To make use of this system, you need a package with a .git_commit_info.ini
     file - e.g. ``myproject/.git_commit_info.ini`` - that might well look like
     this::
 
         # This is an ini file that may contain information about the code state
         [commit hash]
         # The line below may contain a valid hash if it has been substituted
         # during 'git archive'
         archive_subst_hash=$Format:%h$
         # This line may be modified by the install process
         install_hash=
 
     The .git_commit_info file above is also designed to be used with git
     substitution - so you probably also want a ``.gitattributes`` file in the
     root directory of your working tree that contains something like this::
 
        myproject/.git_commit_info.ini export-subst
 
     That will cause the ``.git_commit_info.ini`` file to get filled in by ``git
     archive`` - useful in case someone makes such an archive - for example with
     via the github 'download source' button.
 
     Although all the above will work as is, you might consider having something
     like a ``get_info()`` function in your package to display the commit
     information at the terminal.  See the ``pkg_info.py`` module in the nipy
     package for an example.
     """
     class MyBuildPy(build_cmd):
         ''' Subclass to write commit data into installation tree '''
         def run(self):
             build_cmd.run(self)
             import subprocess
             proc = subprocess.Popen('git rev-parse --short HEAD',
                                     stdout=subprocess.PIPE,
                                     stderr=subprocess.PIPE,
                                     shell=True)
             repo_commit, _ = proc.communicate()
             # We write the installation commit even if it's empty
             cfg_parser = ConfigParser()
             cfg_parser.read(pjoin(pkg_dir, '.git_commit_info.ini'))
             if not cfg_parser.has_section('commit hash'):
                 # just in case the ini file is empty or doesn't exist, somehow
                 # we don't want the next line to raise
                 cfg_parser.add_section('commit hash')
             cfg_parser.set('commit hash', 'install_hash', repo_commit.decode('ascii'))
             out_pth = pjoin(self.build_lib, pkg_dir, '.git_commit_info.ini')
             out_file = open(out_pth, 'wt')
             cfg_parser.write(out_file)
             out_file.close()
     return MyBuildPy