Merge branch 'newapp'

closes gh-503
closes gh-454
closes gh-79
closes gh-456
closes gh-451

IPython/config/profile/README

@@ -0,0 +1,5 b''
+This is the IPython directory.
+
+For more information on configuring IPython, do:
+
+ipython config -h

IPython/core/profileapp.py

@@ -0,0 +1,219 b''
+# encoding: utf-8
+"""
+An application for managing IPython profiles.
+
+To be invoked as the `ipython profile` subcommand.
+
+Authors:
+
+* Min RK
+
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2011  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import logging
+import os
+
+from IPython.config.application import Application, boolean_flag
+from IPython.core.application import (
+    BaseIPythonApplication, base_flags, base_aliases
+)
+from IPython.core.profiledir import ProfileDir
+from IPython.utils.path import get_ipython_dir
+from IPython.utils.traitlets import Unicode, Bool, Dict
+
+#-----------------------------------------------------------------------------
+# Constants
+#-----------------------------------------------------------------------------
+
+create_help = """Create an IPython profile by name
+
+Create an ipython profile directory by its name or
+profile directory path. Profile directories contain
+configuration, log and security related files and are named
+using the convention 'profile_<name>'. By default they are
+located in your ipython directory. Once created, you will
+can edit the configuration files in the profile
+directory to configure IPython. Most users will create a
+profile directory by name,
+`ipython profile create myprofile`, which will put the directory
+in `<ipython_dir>/profile_myprofile`.
+"""
+list_help = """List available IPython profiles
+
+List all available profiles, by profile location, that can
+be found in the current working directly or in the ipython
+directory. Profile directories are named using the convention
+'profile_<profile>'.
+"""
+profile_help = """Manage IPython profiles
+
+Profile directories contain
+configuration, log and security related files and are named
+using the convention 'profile_<name>'. By default they are
+located in your ipython directory.  You can create profiles
+with `ipython profile create <name>`, or see the profiles you
+already have with `ipython profile list`
+
+To get started configuring IPython, simply do:
+
+$> ipython profile create
+
+and IPython will create the default profile in <ipython_dir>/profile_default,
+where you can edit ipython_config.py to start configuring IPython.
+
+"""
+
+#-----------------------------------------------------------------------------
+# Profile Application Class (for `ipython profile` subcommand)
+#-----------------------------------------------------------------------------
+
+
+
+class ProfileList(Application):
+    name = u'ipython-profile'
+    description = list_help
+    
+    aliases = Dict(dict(
+        ipython_dir = 'ProfileList.ipython_dir',
+        log_level = 'Application.log_level',
+    ))
+    flags = Dict(dict(
+        debug = ({'Application' : {'log_level' : 0}},
+            "Set log_level to 0, maximizing log output."
+        )
+    ))
+    ipython_dir = Unicode(get_ipython_dir(), config=True, 
+        help="""
+        The name of the IPython directory. This directory is used for logging
+        configuration (through profiles), history storage, etc. The default
+        is usually $HOME/.ipython. This options can also be specified through
+        the environment variable IPYTHON_DIR.
+        """
+    )
+    
+    def list_profile_dirs(self):
+        # Find the search paths
+        paths = [os.getcwdu(), self.ipython_dir]
+
+        self.log.warn('Searching for IPython profiles in paths: %r' % paths)
+        for path in paths:
+            files = os.listdir(path)
+            for f in files:
+                full_path = os.path.join(path, f)
+                if os.path.isdir(full_path) and f.startswith('profile_'):
+                    profile = f.split('_',1)[-1]
+                    start_cmd = 'ipython profile=%s' % profile
+                    print start_cmd + " ==> " + full_path
+    
+    def start(self):
+        self.list_profile_dirs()
+
+
+create_flags = {}
+create_flags.update(base_flags)
+create_flags.update(boolean_flag('reset', 'ProfileCreate.overwrite', 
+                "reset config files to defaults", "leave existing config files"))
+create_flags.update(boolean_flag('parallel', 'ProfileCreate.parallel', 
+                "Include parallel computing config files", 
+                "Don't include parallel computing config files"))
+
+class ProfileCreate(BaseIPythonApplication):
+    name = u'ipython-profile'
+    description = create_help
+    auto_create = Bool(True, config=False)
+    
+    def _copy_config_files_default(self):
+        return True
+    
+    parallel = Bool(False, config=True,
+        help="whether to include parallel computing config files")
+    def _parallel_changed(self, name, old, new):
+        parallel_files = [   'ipcontroller_config.py', 
+                            'ipengine_config.py', 
+                            'ipcluster_config.py'
+                        ]
+        if new:
+            for cf in parallel_files:
+                self.config_files.append(cf)
+        else:
+            for cf in parallel_files:
+                if cf in self.config_files:
+                    self.config_files.remove(cf)
+    
+    def parse_command_line(self, argv):
+        super(ProfileCreate, self).parse_command_line(argv)
+        # accept positional arg as profile name
+        if self.extra_args:
+            self.profile = self.extra_args[0]
+    
+    flags = Dict(create_flags)
+    
+    aliases = Dict(dict(profile='BaseIPythonApplication.profile'))
+    
+    classes = [ProfileDir]
+    
+    def init_config_files(self):
+        super(ProfileCreate, self).init_config_files()
+        # use local imports, since these classes may import from here
+        from IPython.frontend.terminal.ipapp import TerminalIPythonApp
+        apps = [TerminalIPythonApp]
+        try:
+            from IPython.frontend.qt.console.qtconsoleapp import IPythonQtConsoleApp
+        except ImportError:
+            pass
+        else:
+            apps.append(IPythonQtConsoleApp)
+        if self.parallel:
+            from IPython.parallel.apps.ipcontrollerapp import IPControllerApp
+            from IPython.parallel.apps.ipengineapp import IPEngineApp
+            from IPython.parallel.apps.ipclusterapp import IPClusterStart
+            from IPython.parallel.apps.iploggerapp import IPLoggerApp
+            apps.extend([
+                IPControllerApp,
+                IPEngineApp,
+                IPClusterStart,
+                IPLoggerApp,
+            ])
+        for App in apps:
+            app = App()
+            app.config.update(self.config)
+            app.log = self.log
+            app.overwrite = self.overwrite
+            app.copy_config_files=True
+            app.profile = self.profile
+            app.init_profile_dir()
+            app.init_config_files()
+    
+    def stage_default_config_file(self):
+        pass
+
+class ProfileApp(Application):
+    name = u'ipython-profile'
+    description = profile_help
+    
+    subcommands = Dict(dict(
+        create = (ProfileCreate, "Create a new profile dir with default config files"),
+        list = (ProfileList, "List existing profiles")
+    ))
+    
+    def start(self):
+        if self.subapp is None:
+            print "No subcommand specified. Must specify one of: %s"%(self.subcommands.keys())
+            print
+            self.print_description()
+            self.print_subcommands()
+            self.exit(1)
+        else:
+            return self.subapp.start()
+

IPython/core/profiledir.py

@@ -0,0 +1,206 b''
+# encoding: utf-8
+"""
+An object for managing IPython profile directories.
+
+Authors:
+
+* Brian Granger
+* Fernando Perez
+* Min RK
+
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2011  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import os
+import shutil
+import sys
+
+from IPython.config.configurable import Configurable
+from IPython.config.loader import Config
+from IPython.utils.path import get_ipython_package_dir, expand_path
+from IPython.utils.traitlets import List, Unicode, Bool
+
+#-----------------------------------------------------------------------------
+# Classes and functions
+#-----------------------------------------------------------------------------
+
+
+#-----------------------------------------------------------------------------
+# Module errors
+#-----------------------------------------------------------------------------
+
+class ProfileDirError(Exception):
+    pass
+
+
+#-----------------------------------------------------------------------------
+# Class for managing profile directories
+#-----------------------------------------------------------------------------
+
+class ProfileDir(Configurable):
+    """An object to manage the profile directory and its resources.
+
+    The profile directory is used by all IPython applications, to manage
+    configuration, logging and security.
+
+    This object knows how to find, create and manage these directories. This
+    should be used by any code that wants to handle profiles.
+    """
+
+    security_dir_name = Unicode('security')
+    log_dir_name = Unicode('log')
+    pid_dir_name = Unicode('pid')
+    security_dir = Unicode(u'')
+    log_dir = Unicode(u'')
+    pid_dir = Unicode(u'')
+
+    location = Unicode(u'', config=True,
+        help="""Set the profile location directly. This overrides the logic used by the
+        `profile` option.""",
+        )
+
+    _location_isset = Bool(False) # flag for detecting multiply set location
+
+    def _location_changed(self, name, old, new):
+        if self._location_isset:
+            raise RuntimeError("Cannot set profile location more than once.")
+        self._location_isset = True
+        if not os.path.isdir(new):
+            os.makedirs(new)
+        
+        # ensure config files exist:
+        self.security_dir = os.path.join(new, self.security_dir_name)
+        self.log_dir = os.path.join(new, self.log_dir_name)
+        self.pid_dir = os.path.join(new, self.pid_dir_name)
+        self.check_dirs()
+
+    def _log_dir_changed(self, name, old, new):
+        self.check_log_dir()
+
+    def check_log_dir(self):
+        if not os.path.isdir(self.log_dir):
+            os.mkdir(self.log_dir)
+
+    def _security_dir_changed(self, name, old, new):
+        self.check_security_dir()
+
+    def check_security_dir(self):
+        if not os.path.isdir(self.security_dir):
+            os.mkdir(self.security_dir, 0700)
+        else:
+            os.chmod(self.security_dir, 0700)
+
+    def _pid_dir_changed(self, name, old, new):
+        self.check_pid_dir()
+
+    def check_pid_dir(self):
+        if not os.path.isdir(self.pid_dir):
+            os.mkdir(self.pid_dir, 0700)
+        else:
+            os.chmod(self.pid_dir, 0700)
+
+    def check_dirs(self):
+        self.check_security_dir()
+        self.check_log_dir()
+        self.check_pid_dir()
+
+    def copy_config_file(self, config_file, path=None, overwrite=False):
+        """Copy a default config file into the active profile directory.
+
+        Default configuration files are kept in :mod:`IPython.config.default`.
+        This function moves these from that location to the working profile
+        directory.
+        """
+        dst = os.path.join(self.location, config_file)
+        if os.path.isfile(dst) and not overwrite:
+            return
+        if path is None:
+            path = os.path.join(get_ipython_package_dir(), u'config', u'profile', u'default')
+        src = os.path.join(path, config_file)
+        shutil.copy(src, dst)
+
+    @classmethod
+    def create_profile_dir(cls, profile_dir, config=None):
+        """Create a new profile directory given a full path.
+
+        Parameters
+        ----------
+        profile_dir : str
+            The full path to the profile directory.  If it does exist, it will
+            be used.  If not, it will be created.
+        """
+        return cls(location=profile_dir, config=config)
+
+    @classmethod
+    def create_profile_dir_by_name(cls, path, name=u'default', config=None):
+        """Create a profile dir by profile name and path.
+
+        Parameters
+        ----------
+        path : unicode
+            The path (directory) to put the profile directory in.
+        name : unicode
+            The name of the profile.  The name of the profile directory will
+            be "profile_<profile>".
+        """
+        if not os.path.isdir(path):
+            raise ProfileDirError('Directory not found: %s' % path)
+        profile_dir = os.path.join(path, u'profile_' + name)
+        return cls(location=profile_dir, config=config)
+
+    @classmethod
+    def find_profile_dir_by_name(cls, ipython_dir, name=u'default', config=None):
+        """Find an existing profile dir by profile name, return its ProfileDir.
+
+        This searches through a sequence of paths for a profile dir.  If it
+        is not found, a :class:`ProfileDirError` exception will be raised.
+
+        The search path algorithm is:
+        1. ``os.getcwd()``
+        2. ``ipython_dir``
+
+        Parameters
+        ----------
+        ipython_dir : unicode or str
+            The IPython directory to use.
+        name : unicode or str
+            The name of the profile.  The name of the profile directory
+            will be "profile_<profile>".
+        """
+        dirname = u'profile_' + name
+        paths = [os.getcwdu(), ipython_dir]
+        for p in paths:
+            profile_dir = os.path.join(p, dirname)
+            if os.path.isdir(profile_dir):
+                return cls(location=profile_dir, config=config)
+        else:
+            raise ProfileDirError('Profile directory not found in paths: %s' % dirname)
+
+    @classmethod
+    def find_profile_dir(cls, profile_dir, config=None):
+        """Find/create a profile dir and return its ProfileDir.
+
+        This will create the profile directory if it doesn't exist.
+
+        Parameters
+        ----------
+        profile_dir : unicode or str
+            The path of the profile directory.  This is expanded using
+            :func:`IPython.utils.genutils.expand_path`.
+        """
+        profile_dir = expand_path(profile_dir)
+        if not os.path.isdir(profile_dir):
+            raise ProfileDirError('Profile directory not found: %s' % profile_dir)
+        return cls(location=profile_dir, config=config)
+
+

IPython/core/shellapp.py

@@ -0,0 +1,244 b''
+#!/usr/bin/env python
+# encoding: utf-8
+"""
+A mixin for :class:`~IPython.core.application.Application` classes that
+launch InteractiveShell instances, load extensions, etc.
+
+Authors
+-------
+
+* Min Ragan-Kelley
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2011  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+from __future__ import absolute_import
+
+import os
+import sys
+
+from IPython.config.application import boolean_flag
+from IPython.config.configurable import Configurable
+from IPython.config.loader import Config
+from IPython.utils.path import filefind
+from IPython.utils.traitlets import Unicode, Instance, List
+
+#-----------------------------------------------------------------------------
+# Aliases and Flags
+#-----------------------------------------------------------------------------
+
+shell_flags = {}
+
+addflag = lambda *args: shell_flags.update(boolean_flag(*args))
+addflag('autoindent', 'InteractiveShell.autoindent',
+        'Turn on autoindenting.', 'Turn off autoindenting.'
+)
+addflag('automagic', 'InteractiveShell.automagic',
+        """Turn on the auto calling of magic commands. Type %%magic at the
+        IPython  prompt  for  more information.""",
+        'Turn off the auto calling of magic commands.'
+)
+addflag('pdb', 'InteractiveShell.pdb',
+    "Enable auto calling the pdb debugger after every exception.",
+    "Disable auto calling the pdb debugger after every exception."
+)
+addflag('pprint', 'PlainTextFormatter.pprint',
+    "Enable auto pretty printing of results.",
+    "Disable auto auto pretty printing of results."
+)
+addflag('color-info', 'InteractiveShell.color_info',
+    """IPython can display information about objects via a set of func-
+    tions, and optionally can use colors for this, syntax highlighting
+    source code and various other elements.  However, because this
+    information is passed through a pager (like 'less') and many pagers get
+    confused with color codes, this option is off by default.  You can test
+    it and turn it on permanently in your ipython_config.py file if it
+    works for you.  Test it and turn it on permanently if it works with
+    your system.  The magic function %%color_info allows you to toggle this
+    interactively for testing.""",
+    "Disable using colors for info related things."
+)
+addflag('deep-reload', 'InteractiveShell.deep_reload',
+    """Enable deep (recursive) reloading by default. IPython can use the
+    deep_reload module which reloads changes in modules recursively (it
+    replaces the reload() function, so you don't need to change anything to
+    use it). deep_reload() forces a full reload of modules whose code may
+    have changed, which the default reload() function does not.  When
+    deep_reload is off, IPython will use the normal reload(), but
+    deep_reload will still be available as dreload(). This feature is off
+    by default [which means that you have both normal reload() and
+    dreload()].""",
+    "Disable deep (recursive) reloading by default."
+)
+nosep_config = Config()
+nosep_config.InteractiveShell.separate_in = ''
+nosep_config.InteractiveShell.separate_out = ''
+nosep_config.InteractiveShell.separate_out2 = ''
+
+shell_flags['nosep']=(nosep_config, "Eliminate all spacing between prompts.")
+
+
+# it's possible we don't want short aliases for *all* of these:
+shell_aliases = dict(
+    autocall='InteractiveShell.autocall',
+    cache_size='InteractiveShell.cache_size',
+    colors='InteractiveShell.colors',
+    logfile='InteractiveShell.logfile',
+    log_append='InteractiveShell.logappend',
+    c='InteractiveShellApp.code_to_run',
+    ext='InteractiveShellApp.extra_extension',
+)
+
+#-----------------------------------------------------------------------------
+# Main classes and functions
+#-----------------------------------------------------------------------------
+
+class InteractiveShellApp(Configurable):
+    """A Mixin for applications that start InteractiveShell instances.
+    
+    Provides configurables for loading extensions and executing files
+    as part of configuring a Shell environment.
+    
+    Provides init_extensions() and init_code() methods, to be called
+    after init_shell(), which must be implemented by subclasses.
+    """
+    extensions = List(Unicode, config=True,
+        help="A list of dotted module names of IPython extensions to load."
+    )
+    extra_extension = Unicode('', config=True,
+        help="dotted module name of an IPython extension to load."
+    )
+    def _extra_extension_changed(self, name, old, new):
+        if new:
+            # add to self.extensions
+            self.extensions.append(new)
+
+    exec_files = List(Unicode, config=True,
+        help="""List of files to run at IPython startup."""
+    )
+    file_to_run = Unicode('', config=True,
+        help="""A file to be run""")
+
+    exec_lines = List(Unicode, config=True,
+        help="""lines of code to run at IPython startup."""
+    )
+    code_to_run = Unicode('', config=True,
+        help="Execute the given command string."
+    )
+    shell = Instance('IPython.core.interactiveshell.InteractiveShellABC')
+
+    def init_shell(self):
+        raise NotImplementedError("Override in subclasses")
+    
+    def init_extensions(self):
+        """Load all IPython extensions in IPythonApp.extensions.
+
+        This uses the :meth:`ExtensionManager.load_extensions` to load all
+        the extensions listed in ``self.extensions``.
+        """
+        if not self.extensions:
+            return
+        try:
+            self.log.debug("Loading IPython extensions...")
+            extensions = self.extensions
+            for ext in extensions:
+                try:
+                    self.log.info("Loading IPython extension: %s" % ext)
+                    self.shell.extension_manager.load_extension(ext)
+                except:
+                    self.log.warn("Error in loading extension: %s" % ext)
+                    self.shell.showtraceback()
+        except:
+            self.log.warn("Unknown error in loading extensions:")
+            self.shell.showtraceback()
+
+    def init_code(self):
+        """run the pre-flight code, specified via exec_lines"""
+        self._run_exec_lines()
+        self._run_exec_files()
+        self._run_cmd_line_code()
+
+    def _run_exec_lines(self):
+        """Run lines of code in IPythonApp.exec_lines in the user's namespace."""
+        if not self.exec_lines:
+            return
+        try:
+            self.log.debug("Running code from IPythonApp.exec_lines...")
+            for line in self.exec_lines:
+                try:
+                    self.log.info("Running code in user namespace: %s" %
+                                  line)
+                    self.shell.run_cell(line, store_history=False)
+                except:
+                    self.log.warn("Error in executing line in user "
+                                  "namespace: %s" % line)
+                    self.shell.showtraceback()
+        except:
+            self.log.warn("Unknown error in handling IPythonApp.exec_lines:")
+            self.shell.showtraceback()
+
+    def _exec_file(self, fname):
+        full_filename = filefind(fname, [u'.', self.ipython_dir])
+        if os.path.isfile(full_filename):
+            if full_filename.endswith(u'.py'):
+                self.log.info("Running file in user namespace: %s" %
+                              full_filename)
+                # Ensure that __file__ is always defined to match Python behavior
+                self.shell.user_ns['__file__'] = fname
+                try:
+                    self.shell.safe_execfile(full_filename, self.shell.user_ns)
+                finally:
+                    del self.shell.user_ns['__file__']
+            elif full_filename.endswith('.ipy'):
+                self.log.info("Running file in user namespace: %s" %
+                              full_filename)
+                self.shell.safe_execfile_ipy(full_filename)
+            else:
+                self.log.warn("File does not have a .py or .ipy extension: <%s>"
+                               % full_filename)
+
+    def _run_exec_files(self):
+        """Run files from IPythonApp.exec_files"""
+        if not self.exec_files:
+            return
+
+        self.log.debug("Running files in IPythonApp.exec_files...")
+        try:
+            for fname in self.exec_files:
+                self._exec_file(fname)
+        except:
+            self.log.warn("Unknown error in handling IPythonApp.exec_files:")
+            self.shell.showtraceback()
+
+    def _run_cmd_line_code(self):
+        """Run code or file specified at the command-line"""
+        if self.code_to_run:
+            line = self.code_to_run
+            try:
+                self.log.info("Running code given at command line (c=): %s" %
+                              line)
+                self.shell.run_cell(line, store_history=False)
+            except:
+                self.log.warn("Error in executing line in user namespace: %s" %
+                              line)
+                self.shell.showtraceback()
+
+        # Like Python itself, ignore the second if the first of these is present
+        elif self.file_to_run:
+            fname = self.file_to_run
+            try:
+                self._exec_file(fname)
+            except:
+                self.log.warn("Error in executing file in user namespace: %s" %
+                              fname)
+                self.shell.showtraceback()
+

IPython/parallel/apps/baseapp.py

@@ -0,0 +1,267 b''
+#!/usr/bin/env python
+# encoding: utf-8
+"""
+The Base Application class for IPython.parallel apps
+
+Authors:
+
+* Brian Granger
+* Min RK
+
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2011  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+from __future__ import with_statement
+
+import os
+import logging
+import re
+import sys
+
+from subprocess import Popen, PIPE
+
+from IPython.core import release
+from IPython.core.crashhandler import CrashHandler
+from IPython.core.application import (
+    BaseIPythonApplication,
+    base_aliases as base_ip_aliases,
+    base_flags as base_ip_flags
+)
+from IPython.utils.path import expand_path
+
+from IPython.utils.traitlets import Unicode, Bool, Instance, Dict, List
+
+#-----------------------------------------------------------------------------
+# Module errors
+#-----------------------------------------------------------------------------
+
+class PIDFileError(Exception):
+    pass
+
+
+#-----------------------------------------------------------------------------
+# Crash handler for this application
+#-----------------------------------------------------------------------------
+
+
+_message_template = """\
+Oops, $self.app_name crashed. We do our best to make it stable, but...
+
+A crash report was automatically generated with the following information:
+  - A verbatim copy of the crash traceback.
+  - Data on your current $self.app_name configuration.
+
+It was left in the file named:
+\t'$self.crash_report_fname'
+If you can email this file to the developers, the information in it will help
+them in understanding and correcting the problem.
+
+You can mail it to: $self.contact_name at $self.contact_email
+with the subject '$self.app_name Crash Report'.
+
+If you want to do it now, the following command will work (under Unix):
+mail -s '$self.app_name Crash Report' $self.contact_email < $self.crash_report_fname
+
+To ensure accurate tracking of this issue, please file a report about it at:
+$self.bug_tracker
+"""
+
+class ParallelCrashHandler(CrashHandler):
+    """sys.excepthook for IPython itself, leaves a detailed report on disk."""
+
+    message_template = _message_template
+
+    def __init__(self, app):
+        contact_name = release.authors['Min'][0]
+        contact_email = release.authors['Min'][1]
+        bug_tracker = 'http://github.com/ipython/ipython/issues'
+        super(ParallelCrashHandler,self).__init__(
+            app, contact_name, contact_email, bug_tracker
+        )
+
+
+#-----------------------------------------------------------------------------
+# Main application
+#-----------------------------------------------------------------------------
+base_aliases = {}
+base_aliases.update(base_ip_aliases)
+base_aliases.update({
+    'profile_dir' : 'ProfileDir.location',
+    'log_level' : 'BaseParallelApplication.log_level',
+    'work_dir' : 'BaseParallelApplication.work_dir',
+    'log_to_file' : 'BaseParallelApplication.log_to_file',
+    'clean_logs' : 'BaseParallelApplication.clean_logs',
+    'log_url' : 'BaseParallelApplication.log_url',
+})
+
+base_flags = {
+    'log-to-file' : (
+        {'BaseParallelApplication' : {'log_to_file' : True}},
+        "send log output to a file"
+    )
+}
+base_flags.update(base_ip_flags)
+
+class BaseParallelApplication(BaseIPythonApplication):
+    """The base Application for IPython.parallel apps
+    
+    Principle extensions to BaseIPyythonApplication:
+    
+    * work_dir
+    * remote logging via pyzmq
+    * IOLoop instance
+    """
+
+    crash_handler_class = ParallelCrashHandler
+    
+    def _log_level_default(self):
+        # temporarily override default_log_level to INFO
+        return logging.INFO
+
+    work_dir = Unicode(os.getcwdu(), config=True,
+        help='Set the working dir for the process.'
+    )
+    def _work_dir_changed(self, name, old, new):
+        self.work_dir = unicode(expand_path(new))
+
+    log_to_file = Bool(config=True,
+        help="whether to log to a file")
+
+    clean_logs = Bool(False, config=True,
+        help="whether to cleanup old logfiles before starting")
+
+    log_url = Unicode('', config=True,
+        help="The ZMQ URL of the iplogger to aggregate logging.")
+
+    def _config_files_default(self):
+        return ['ipcontroller_config.py', 'ipengine_config.py', 'ipcluster_config.py']
+    
+    loop = Instance('zmq.eventloop.ioloop.IOLoop')
+    def _loop_default(self):
+        from zmq.eventloop.ioloop import IOLoop
+        return IOLoop.instance()
+
+    aliases = Dict(base_aliases)
+    flags = Dict(base_flags)
+    
+    def initialize(self, argv=None):
+        """initialize the app"""
+        super(BaseParallelApplication, self).initialize(argv)
+        self.to_work_dir()
+        self.reinit_logging()
+        
+    def to_work_dir(self):
+        wd = self.work_dir
+        if unicode(wd) != os.getcwdu():
+            os.chdir(wd)
+            self.log.info("Changing to working dir: %s" % wd)
+        # This is the working dir by now.
+        sys.path.insert(0, '')
+
+    def reinit_logging(self):
+        # Remove old log files
+        log_dir = self.profile_dir.log_dir
+        if self.clean_logs:
+            for f in os.listdir(log_dir):
+                if re.match(r'%s-\d+\.(log|err|out)'%self.name,f):
+                    os.remove(os.path.join(log_dir, f))
+        if self.log_to_file:
+            # Start logging to the new log file
+            log_filename = self.name + u'-' + str(os.getpid()) + u'.log'
+            logfile = os.path.join(log_dir, log_filename)
+            open_log_file = open(logfile, 'w')
+        else:
+            open_log_file = None
+        if open_log_file is not None:
+            self.log.removeHandler(self._log_handler)
+            self._log_handler = logging.StreamHandler(open_log_file)
+            self._log_formatter = logging.Formatter("[%(name)s] %(message)s")
+            self._log_handler.setFormatter(self._log_formatter)
+            self.log.addHandler(self._log_handler)
+
+    def write_pid_file(self, overwrite=False):
+        """Create a .pid file in the pid_dir with my pid.
+
+        This must be called after pre_construct, which sets `self.pid_dir`.
+        This raises :exc:`PIDFileError` if the pid file exists already.
+        """
+        pid_file = os.path.join(self.profile_dir.pid_dir, self.name + u'.pid')
+        if os.path.isfile(pid_file):
+            pid = self.get_pid_from_file()
+            if not overwrite:
+                raise PIDFileError(
+                    'The pid file [%s] already exists. \nThis could mean that this '
+                    'server is already running with [pid=%s].' % (pid_file, pid)
+                )
+        with open(pid_file, 'w') as f:
+            self.log.info("Creating pid file: %s" % pid_file)
+            f.write(repr(os.getpid())+'\n')
+
+    def remove_pid_file(self):
+        """Remove the pid file.
+
+        This should be called at shutdown by registering a callback with
+        :func:`reactor.addSystemEventTrigger`. This needs to return
+        ``None``.
+        """
+        pid_file = os.path.join(self.profile_dir.pid_dir, self.name + u'.pid')
+        if os.path.isfile(pid_file):
+            try:
+                self.log.info("Removing pid file: %s" % pid_file)
+                os.remove(pid_file)
+            except:
+                self.log.warn("Error removing the pid file: %s" % pid_file)
+
+    def get_pid_from_file(self):
+        """Get the pid from the pid file.
+
+        If the  pid file doesn't exist a :exc:`PIDFileError` is raised.
+        """
+        pid_file = os.path.join(self.profile_dir.pid_dir, self.name + u'.pid')
+        if os.path.isfile(pid_file):
+            with open(pid_file, 'r') as f:
+                s = f.read().strip()
+                try:
+                    pid = int(s)
+                except:
+                    raise PIDFileError("invalid pid file: %s (contents: %r)"%(pid_file, s))
+                return pid
+        else:
+            raise PIDFileError('pid file not found: %s' % pid_file)
+    
+    def check_pid(self, pid):
+        if os.name == 'nt':
+            try:
+                import ctypes
+                # returns 0 if no such process (of ours) exists
+                # positive int otherwise
+                p = ctypes.windll.kernel32.OpenProcess(1,0,pid)
+            except Exception:
+                self.log.warn(
+                    "Could not determine whether pid %i is running via `OpenProcess`. "
+                    " Making the likely assumption that it is."%pid
+                )
+                return True
+            return bool(p)
+        else:
+            try:
+                p = Popen(['ps','x'], stdout=PIPE, stderr=PIPE)
+                output,_ = p.communicate()
+            except OSError:
+                self.log.warn(
+                    "Could not determine whether pid %i is running via `ps x`. "
+                    " Making the likely assumption that it is."%pid
+                )
+                return True
+            pids = map(int, re.findall(r'^\W*\d+', output, re.MULTILINE))
+            return pid in pids

IPython/utils/daemonize.py

@@ -0,0 +1,26 b''
+"""daemonize function from twisted.scripts._twistd_unix."""
+
+#-----------------------------------------------------------------------------
+# Copyright (c) Twisted Matrix Laboratories.
+# See Twisted's LICENSE for details.
+# http://twistedmatrix.com/
+#-----------------------------------------------------------------------------
+
+import os, errno
+
+def daemonize():
+    # See http://www.erlenstar.demon.co.uk/unix/faq_toc.html#TOC16
+    if os.fork():   # launch child and...
+        os._exit(0) # kill off parent
+    os.setsid()
+    if os.fork():   # launch child and...
+        os._exit(0) # kill off parent again.
+    null = os.open('/dev/null', os.O_RDWR)
+    for i in range(3):
+        try:
+            os.dup2(null, i)
+        except OSError, e:
+            if e.errno != errno.EBADF:
+                raise
+    os.close(null)
+

IPython/zmq/kernelapp.py

@@ -0,0 +1,215 b''
+#!/usr/bin/env python
+"""An Application for launching a kernel
+
+Authors
+-------
+* MinRK
+"""
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2011  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING.txt, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+# Standard library imports.
+import os
+import sys
+
+# System library imports.
+import zmq
+
+# IPython imports.
+from IPython.core.ultratb import FormattedTB
+from IPython.core.application import (
+    BaseIPythonApplication, base_flags, base_aliases
+)
+from IPython.utils import io
+from IPython.utils.localinterfaces import LOCALHOST
+from IPython.utils.traitlets import Any, Instance, Dict, Unicode, Int, Bool
+from IPython.utils.importstring import import_item
+# local imports
+from IPython.zmq.heartbeat import Heartbeat
+from IPython.zmq.parentpoller import ParentPollerUnix, ParentPollerWindows
+from IPython.zmq.session import Session
+
+
+#-----------------------------------------------------------------------------
+# Flags and Aliases
+#-----------------------------------------------------------------------------
+
+kernel_aliases = dict(base_aliases)
+kernel_aliases.update({
+    'ip' : 'KernelApp.ip',
+    'hb' : 'KernelApp.hb_port',
+    'shell' : 'KernelApp.shell_port',
+    'iopub' : 'KernelApp.iopub_port',
+    'stdin' : 'KernelApp.stdin_port',
+    'parent': 'KernelApp.parent',
+})
+if sys.platform.startswith('win'):
+    kernel_aliases['interrupt'] = 'KernelApp.interrupt'
+
+kernel_flags = dict(base_flags)
+kernel_flags.update({
+    'no-stdout' : (
+            {'KernelApp' : {'no_stdout' : True}},
+            "redirect stdout to the null device"),
+    'no-stderr' : (
+            {'KernelApp' : {'no_stderr' : True}},
+            "redirect stderr to the null device"),
+})
+
+
+#-----------------------------------------------------------------------------
+# Application class for starting a Kernel
+#-----------------------------------------------------------------------------
+
+class KernelApp(BaseIPythonApplication):
+    name='pykernel'
+    aliases = Dict(kernel_aliases)
+    flags = Dict(kernel_flags)
+    classes = [Session]
+    # the kernel class, as an importstring
+    kernel_class = Unicode('IPython.zmq.pykernel.Kernel')
+    kernel = Any()
+    poller = Any() # don't restrict this even though current pollers are all Threads
+    heartbeat = Instance(Heartbeat)
+    session = Instance('IPython.zmq.session.Session')
+    ports = Dict()
+
+    # connection info:
+    ip = Unicode(LOCALHOST, config=True,
+        help="Set the IP or interface on which the kernel will listen.")
+    hb_port = Int(0, config=True, help="set the heartbeat port [default: random]")
+    shell_port = Int(0, config=True, help="set the shell (XREP) port [default: random]")
+    iopub_port = Int(0, config=True, help="set the iopub (PUB) port [default: random]")
+    stdin_port = Int(0, config=True, help="set the stdin (XREQ) port [default: random]")
+
+    # streams, etc.
+    no_stdout = Bool(False, config=True, help="redirect stdout to the null device")
+    no_stderr = Bool(False, config=True, help="redirect stderr to the null device")
+    outstream_class = Unicode('IPython.zmq.iostream.OutStream', config=True,
+        help="The importstring for the OutStream factory")
+    displayhook_class = Unicode('IPython.zmq.displayhook.DisplayHook', config=True,
+        help="The importstring for the DisplayHook factory")
+
+    # polling
+    parent = Int(0, config=True,
+        help="""kill this process if its parent dies.  On Windows, the argument
+        specifies the HANDLE of the parent process, otherwise it is simply boolean.
+        """)
+    interrupt = Int(0, config=True,
+        help="""ONLY USED ON WINDOWS
+        Interrupt this process when the parent is signalled.
+        """)
+
+    def init_crash_handler(self):
+        # Install minimal exception handling
+        sys.excepthook = FormattedTB(mode='Verbose', color_scheme='NoColor',
+                                     ostream=sys.__stdout__)
+
+    def init_poller(self):
+        if sys.platform == 'win32':
+            if self.interrupt or self.parent:
+                self.poller = ParentPollerWindows(self.interrupt, self.parent)
+        elif self.parent:
+            self.poller = ParentPollerUnix()
+
+    def _bind_socket(self, s, port):
+        iface = 'tcp://%s' % self.ip
+        if port <= 0:
+            port = s.bind_to_random_port(iface)
+        else:
+            s.bind(iface + ':%i'%port)
+        return port
+
+    def init_sockets(self):
+        # Create a context, a session, and the kernel sockets.
+        io.raw_print("Starting the kernel at pid:", os.getpid())
+        context = zmq.Context.instance()
+        # Uncomment this to try closing the context.
+        # atexit.register(context.term)
+
+        self.shell_socket = context.socket(zmq.XREP)
+        self.shell_port = self._bind_socket(self.shell_socket, self.shell_port)
+        self.log.debug("shell XREP Channel on port: %i"%self.shell_port)
+
+        self.iopub_socket = context.socket(zmq.PUB)
+        self.iopub_port = self._bind_socket(self.iopub_socket, self.iopub_port)
+        self.log.debug("iopub PUB Channel on port: %i"%self.iopub_port)
+
+        self.stdin_socket = context.socket(zmq.XREQ)
+        self.stdin_port = self._bind_socket(self.stdin_socket, self.stdin_port)
+        self.log.debug("stdin XREQ Channel on port: %i"%self.stdin_port)
+
+        self.heartbeat = Heartbeat(context, (self.ip, self.hb_port))
+        self.hb_port = self.heartbeat.port
+        self.log.debug("Heartbeat REP Channel on port: %i"%self.hb_port)
+
+        # Helper to make it easier to connect to an existing kernel, until we have
+        # single-port connection negotiation fully implemented.
+        self.log.info("To connect another client to this kernel, use:")
+        self.log.info("--external shell={0} iopub={1} stdin={2} hb={3}".format(
+            self.shell_port, self.iopub_port, self.stdin_port, self.hb_port))
+
+
+        self.ports = dict(shell=self.shell_port, iopub=self.iopub_port,
+                                stdin=self.stdin_port, hb=self.hb_port)
+
+    def init_session(self):
+        """create our session object"""
+        self.session = Session(config=self.config, username=u'kernel')
+
+    def init_io(self):
+        """redirects stdout/stderr, and installs a display hook"""
+        # Re-direct stdout/stderr, if necessary.
+        if self.no_stdout or self.no_stderr:
+            blackhole = file(os.devnull, 'w')
+            if self.no_stdout:
+                sys.stdout = sys.__stdout__ = blackhole
+            if self.no_stderr:
+                sys.stderr = sys.__stderr__ = blackhole
+
+        # Redirect input streams and set a display hook.
+
+        if self.outstream_class:
+            outstream_factory = import_item(str(self.outstream_class))
+            sys.stdout = outstream_factory(self.session, self.iopub_socket, u'stdout')
+            sys.stderr = outstream_factory(self.session, self.iopub_socket, u'stderr')
+        if self.displayhook_class:
+            displayhook_factory = import_item(str(self.displayhook_class))
+            sys.displayhook = displayhook_factory(self.session, self.iopub_socket)
+
+    def init_kernel(self):
+        """Create the Kernel object itself"""
+        kernel_factory = import_item(str(self.kernel_class))
+        self.kernel = kernel_factory(config=self.config, session=self.session,
+                                shell_socket=self.shell_socket,
+                                iopub_socket=self.iopub_socket,
+                                stdin_socket=self.stdin_socket,
+                                log=self.log
+        )
+        self.kernel.record_ports(self.ports)
+
+    def initialize(self, argv=None):
+        super(KernelApp, self).initialize(argv)
+        self.init_session()
+        self.init_poller()
+        self.init_sockets()
+        self.init_io()
+        self.init_kernel()
+
+    def start(self):
+        self.heartbeat.start()
+        if self.poller is not None:
+            self.poller.start()
+        try:
+            self.kernel.start()
+        except KeyboardInterrupt:
+            pass
+

IPython/zmq/logtopics.rst

@@ -0,0 +1,13 b''
+=======================
+Log Topic Specification
+=======================
+
+we use pyzmq to broadcast log events over a PUB socket. Engines, Controllers, etc. can all
+broadcast. SUB sockets can be used to view the logs, and ZMQ topics are used to help
+select out what to follow.
+
+the PUBHandler object that emits the logs can ascribe topics to log messages. The order is:
+
+<root_topic>.<loglevel>.<subtopic>[.<etc>]
+
+root_topic is specified as an attribute

IPython/config/application.py

@@ -5,6 +5,7 b' A base class for a configurable application.'
 Authors:
 
 * Brian Granger
+* Min RK
 """
 
 #-----------------------------------------------------------------------------
@@ -18,22 +19,29 b' Authors:'
 # Imports
 #-----------------------------------------------------------------------------
 
-from copy import deepcopy
 import logging
+import os
+import re
 import sys
+from copy import deepcopy
 
 from IPython.config.configurable import SingletonConfigurable
 from IPython.config.loader import (
-    KeyValueConfigLoader, PyFileConfigLoader, Config
+    KeyValueConfigLoader, PyFileConfigLoader, Config, ArgumentError
 )
 
 from IPython.utils.traitlets import (
-    Unicode, List, Int, Enum, Dict
+    Unicode, List, Int, Enum, Dict, Instance
 )
-from IPython.utils.text import indent
+from IPython.utils.importstring import import_item
+from IPython.utils.text import indent, wrap_paragraphs, dedent
+
+#-----------------------------------------------------------------------------
+# function for re-wrapping a helpstring
+#-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
-# Descriptions for
+# Descriptions for the various sections
 #-----------------------------------------------------------------------------
 
 flag_description = """
@@ -41,6 +49,8 b" Flags are command-line arguments passed as '--<flag>'."
 These take no parameters, unlike regular key-value arguments.
 They are typically used for setting boolean flags, or enabling
 modes that involve setting multiple options together.
+
+Flags *always* begin with '--', never just one '-'.
 """.strip() # trim newlines of front and back
 
 alias_description = """
@@ -61,12 +71,16 b' This line is evaluated in Python, so simple expressions are allowed, e.g.'
 #-----------------------------------------------------------------------------
 
 
+class ApplicationError(Exception):
+    pass
+
+
 class Application(SingletonConfigurable):
     """A singleton application with full configuration support."""
 
     # The name of the application, will usually match the name of the command
     # line application
-    app_name = Unicode(u'application')
+    name = Unicode(u'application')
 
     # The description of the application that is printed at the beginning
     # of the help.
@@ -85,9 +99,16 b' class Application(SingletonConfigurable):'
     version = Unicode(u'0.0')
 
     # The log level for the application
-    log_level = Enum((0,10,20,30,40,50), default_value=logging.WARN,
-                     config=True,
-                     help="Set the log level (0,10,20,30,40,50).")
+    log_level = Enum((0,10,20,30,40,50,'DEBUG','INFO','WARN','ERROR','CRITICAL'),
+                    default_value=logging.WARN,
+                    config=True,
+                    help="Set the log level by value or name.")
+    def _log_level_changed(self, name, old, new):
+        """Adjust the log level when log_level is set."""
+        if isinstance(new, basestring):
+            new = getattr(logging, new)
+            self.log_level = new
+        self.log.setLevel(new)
     
     # the alias map for configurables
     aliases = Dict(dict(log_level='Application.log_level'))
@@ -97,6 +118,25 b' class Application(SingletonConfigurable):'
     # this must be a dict of two-tuples, the first element being the Config/dict
     # and the second being the help string for the flag
     flags = Dict()
+    def _flags_changed(self, name, old, new):
+        """ensure flags dict is valid"""
+        for key,value in new.iteritems():
+            assert len(value) == 2, "Bad flag: %r:%s"%(key,value)
+            assert isinstance(value[0], (dict, Config)), "Bad flag: %r:%s"%(key,value)
+            assert isinstance(value[1], basestring), "Bad flag: %r:%s"%(key,value)
+        
+    
+    # subcommands for launching other applications
+    # if this is not empty, this will be a parent Application
+    # this must be a dict of two-tuples, 
+    # the first element being the application class/import string
+    # and the second being the help string for the subcommand
+    subcommands = Dict()
+    # parse_command_line will initialize a subapp, if requested
+    subapp = Instance('IPython.config.application.Application', allow_none=True)
+    
+    # extra command-line arguments that don't set config values
+    extra_args = List(Unicode)
     
 
     def __init__(self, **kwargs):
@@ -105,13 +145,13 b' class Application(SingletonConfigurable):'
         # options.
         self.classes.insert(0, self.__class__)
         
-        # ensure self.flags dict is valid
-        for key,value in self.flags.iteritems():
-            assert len(value) == 2, "Bad flag: %r:%s"%(key,value)
-            assert isinstance(value[0], (dict, Config)), "Bad flag: %r:%s"%(key,value)
-            assert isinstance(value[1], basestring), "Bad flag: %r:%s"%(key,value)
         self.init_logging()
-    
+
+    def _config_changed(self, name, old, new):
+        SingletonConfigurable._config_changed(self, name, old, new)
+        self.log.debug('Config changed:')
+        self.log.debug(repr(new))
+
     def init_logging(self):
         """Start logging for this application.
 
@@ -125,69 +165,114 b' class Application(SingletonConfigurable):'
         self._log_formatter = logging.Formatter("[%(name)s] %(message)s")
         self._log_handler.setFormatter(self._log_formatter)
         self.log.addHandler(self._log_handler)
-
-    def _log_level_changed(self, name, old, new):
-        """Adjust the log level when log_level is set."""
-        self.log.setLevel(new)
+    
+    def initialize(self, argv=None):
+        """Do the basic steps to configure me.
+        
+        Override in subclasses.
+        """
+        self.parse_command_line(argv)
+        
+    
+    def start(self):
+        """Start the app mainloop.
+        
+        Override in subclasses.
+        """
+        if self.subapp is not None:
+            return self.subapp.start()
     
     def print_alias_help(self):
-        """print the alias part of the help"""
+        """Print the alias part of the help."""
         if not self.aliases:
             return
-            
-        print "Aliases"
-        print "-------"
-        print self.alias_description
-        print
+        
+        lines = ['Aliases']
+        lines.append('-'*len(lines[0]))
+        lines.append('')
+        for p in wrap_paragraphs(self.alias_description):
+            lines.append(p)
+            lines.append('')
         
         classdict = {}
-        for c in self.classes:
-            classdict[c.__name__] = c
+        for cls in self.classes:
+            # include all parents (up to, but excluding Configurable) in available names
+            for c in cls.mro()[:-3]:
+                classdict[c.__name__] = c
         
         for alias, longname in self.aliases.iteritems():
             classname, traitname = longname.split('.',1)
             cls = classdict[classname]
             
             trait = cls.class_traits(config=True)[traitname]
-            help = trait.get_metadata('help')
-            print alias, "(%s)"%longname, ':', trait.__class__.__name__
-            if help:
-                print indent(help)
-        print
+            help = cls.class_get_trait_help(trait)
+            help = help.replace(longname, "%s (%s)"%(alias, longname), 1)
+            lines.append(help)
+        lines.append('')
+        print '\n'.join(lines)
     
     def print_flag_help(self):
-        """print the flag part of the help"""
+        """Print the flag part of the help."""
         if not self.flags:
             return
         
-        print "Flags"
-        print "-----"
-        print self.flag_description
-        print
+        lines = ['Flags']
+        lines.append('-'*len(lines[0]))
+        lines.append('')
+        for p in wrap_paragraphs(self.flag_description):
+            lines.append(p)
+            lines.append('')
         
         for m, (cfg,help) in self.flags.iteritems():
-            print '--'+m
-            print indent(help)
-        print
+            lines.append('--'+m)
+            lines.append(indent(dedent(help.strip())))
+        lines.append('')
+        print '\n'.join(lines)
     
-    def print_help(self):
-        """Print the help for each Configurable class in self.classes."""
+    def print_subcommands(self):
+        """Print the subcommand part of the help."""
+        if not self.subcommands:
+            return
+        
+        lines = ["Subcommands"]
+        lines.append('-'*len(lines[0]))
+        for subc, (cls,help) in self.subcommands.iteritems():
+            lines.append("%s : %s"%(subc, cls))
+            if help:
+                lines.append(indent(dedent(help.strip())))
+        lines.append('')
+        print '\n'.join(lines)
+    
+    def print_help(self, classes=False):
+        """Print the help for each Configurable class in self.classes.
+        
+        If classes=False (the default), only flags and aliases are printed.
+        """
+        self.print_subcommands()
         self.print_flag_help()
         self.print_alias_help()
-        if self.classes:
-            print "Class parameters"
-            print "----------------"
-            print self.keyvalue_description
-            print
         
-        for cls in self.classes:
-            cls.class_print_help()
+        if classes:
+            if self.classes:
+                print "Class parameters"
+                print "----------------"
+                print
+                for p in wrap_paragraphs(self.keyvalue_description):
+                    print p
+                    print
+        
+            for cls in self.classes:
+                cls.class_print_help()
+                print
+        else:
+            print "To see all available configurables, use `--help-all`"
             print
 
     def print_description(self):
         """Print the application description."""
-        print self.description
-        print
+        for p in wrap_paragraphs(self.description):
+            print p
+            print
 
     def print_version(self):
         """Print the version string."""
@@ -201,30 +286,108 b' class Application(SingletonConfigurable):'
         newconfig._merge(config)
         # Save the combined config as self.config, which triggers the traits
         # events.
-        self.config = config
-
+        self.config = newconfig
+    
+    def initialize_subcommand(self, subc, argv=None):
+        """Initialize a subcommand with argv."""
+        subapp,help = self.subcommands.get(subc)
+        
+        if isinstance(subapp, basestring):
+            subapp = import_item(subapp)
+        
+        # clear existing instances
+        self.__class__.clear_instance()
+        # instantiate
+        self.subapp = subapp.instance()
+        # and initialize subapp
+        self.subapp.initialize(argv)
+        
     def parse_command_line(self, argv=None):
         """Parse the command line arguments."""
         argv = sys.argv[1:] if argv is None else argv
 
-        if '-h' in argv or '--help' in argv:
+        if self.subcommands and len(argv) > 0:
+            # we have subcommands, and one may have been specified
+            subc, subargv = argv[0], argv[1:]
+            if re.match(r'^\w(\-?\w)*$', subc) and subc in self.subcommands:
+                # it's a subcommand, and *not* a flag or class parameter
+                return self.initialize_subcommand(subc, subargv)
+            
+        if '-h' in argv or '--help' in argv or '--help-all' in argv:
             self.print_description()
-            self.print_help()
-            sys.exit(1)
+            self.print_help('--help-all' in argv)
+            self.exit(0)
 
         if '--version' in argv:
             self.print_version()
-            sys.exit(1)
+            self.exit(0)
         
         loader = KeyValueConfigLoader(argv=argv, aliases=self.aliases,
                                         flags=self.flags)
-        config = loader.load_config()
+        try:
+            config = loader.load_config()
+        except ArgumentError as e:
+            self.log.fatal(str(e))
+            self.print_description()
+            self.print_help()
+            self.exit(1)
         self.update_config(config)
+        # store unparsed args in extra_args
+        self.extra_args = loader.extra_args
 
     def load_config_file(self, filename, path=None):
         """Load a .py based config file by filename and path."""
-        # TODO: this raises IOError if filename does not exist.
         loader = PyFileConfigLoader(filename, path=path)
         config = loader.load_config()
         self.update_config(config)
+    
+    def generate_config_file(self):
+        """generate default config file from Configurables"""
+        lines = ["# Configuration file for %s."%self.name]
+        lines.append('')
+        lines.append('c = get_config()')
+        lines.append('')
+        for cls in self.classes:
+            lines.append(cls.class_config_section())
+        return '\n'.join(lines)
+
+    def exit(self, exit_status=0):
+        self.log.debug("Exiting application: %s" % self.name)
+        sys.exit(exit_status)
+
+#-----------------------------------------------------------------------------
+# utility functions, for convenience
+#-----------------------------------------------------------------------------
+
+def boolean_flag(name, configurable, set_help='', unset_help=''):
+    """Helper for building basic --trait, --no-trait flags.
+    
+    Parameters
+    ----------
+    
+    name : str
+        The name of the flag.
+    configurable : str
+        The 'Class.trait' string of the trait to be set/unset with the flag
+    set_help : unicode
+        help string for --name flag
+    unset_help : unicode
+        help string for --no-name flag
+    
+    Returns
+    -------
+    
+    cfg : dict
+        A dict with two keys: 'name', and 'no-name', for setting and unsetting
+        the trait, respectively.
+    """
+    # default helpstrings
+    set_help = set_help or "set %s=True"%configurable
+    unset_help = unset_help or "set %s=False"%configurable
+    
+    cls,trait = configurable.split('.')
+    
+    setter = {cls : {trait : True}}
+    unsetter = {cls : {trait : False}}
+    return {name : (setter, set_help), 'no-'+name : (unsetter, unset_help)}
 

IPython/config/configurable.py

@@ -7,10 +7,11 b' Authors:'
 
 * Brian Granger
 * Fernando Perez
+* Min RK
 """
 
 #-----------------------------------------------------------------------------
-#  Copyright (C) 2008-2010  The IPython Development Team
+#  Copyright (C) 2008-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
@@ -20,12 +21,12 b' Authors:'
 # Imports
 #-----------------------------------------------------------------------------
 
-from copy import deepcopy
 import datetime
+from copy import deepcopy
 
 from loader import Config
 from IPython.utils.traitlets import HasTraits, Instance
-from IPython.utils.text import indent
+from IPython.utils.text import indent, wrap_paragraphs
 
 
 #-----------------------------------------------------------------------------
@@ -44,14 +45,13 b' class MultipleInstanceError(ConfigurableError):'
 # Configurable implementation
 #-----------------------------------------------------------------------------
 
-
 class Configurable(HasTraits):
 
     config = Instance(Config,(),{})
     created = None
 
     def __init__(self, **kwargs):
-        """Create a conigurable given a config config.
+        """Create a configurable given a config config.
 
         Parameters
         ----------
@@ -146,18 +146,72 b' class Configurable(HasTraits):'
         final_help = []
         final_help.append(u'%s options' % cls.__name__)
         final_help.append(len(final_help[0])*u'-')
-        for k, v in cls_traits.items():
-            help = v.get_metadata('help')
-            header = "%s.%s : %s" % (cls.__name__, k, v.__class__.__name__)
-            final_help.append(header)
-            if help is not None:
-                final_help.append(indent(help))
+        for k,v in cls.class_traits(config=True).iteritems():
+            help = cls.class_get_trait_help(v)
+            final_help.append(help)
         return '\n'.join(final_help)
+    
+    @classmethod
+    def class_get_trait_help(cls, trait):
+        """Get the help string for a single trait."""
+        lines = []
+        header = "%s.%s : %s" % (cls.__name__, trait.name, trait.__class__.__name__)
+        lines.append(header)
+        try:
+            dvr = repr(trait.get_default_value())
+        except Exception:
+            dvr = None # ignore defaults we can't construct
+        if dvr is not None:
+            if len(dvr) > 64:
+                dvr = dvr[:61]+'...'
+            lines.append(indent('Default: %s'%dvr, 4))
+        if 'Enum' in trait.__class__.__name__:
+            # include Enum choices
+            lines.append(indent('Choices: %r'%(trait.values,)))
+        
+        help = trait.get_metadata('help')
+        if help is not None:
+            help = '\n'.join(wrap_paragraphs(help, 76))
+            lines.append(indent(help, 4))
+        return '\n'.join(lines)
 
     @classmethod
     def class_print_help(cls):
+        """Get the help string for a single trait and print it."""
         print cls.class_get_help()
 
+    @classmethod
+    def class_config_section(cls):
+        """Get the config class config section"""
+        def c(s):
+            """return a commented, wrapped block."""
+            s = '\n\n'.join(wrap_paragraphs(s, 78))
+            
+            return '# ' + s.replace('\n', '\n# ')
+        
+        # section header
+        breaker = '#' + '-'*78
+        s = "# %s configuration"%cls.__name__
+        lines = [breaker, s, breaker, '']
+        # get the description trait
+        desc = cls.class_traits().get('description')
+        if desc:
+            desc = desc.default_value
+        else:
+            # no description trait, use __doc__
+            desc = getattr(cls, '__doc__', '')
+        if desc:
+            lines.append(c(desc))
+            lines.append('')
+        
+        for name,trait in cls.class_traits(config=True).iteritems():
+            help = trait.get_metadata('help') or ''
+            lines.append(c(help))
+            lines.append('# c.%s.%s = %r'%(cls.__name__, name, trait.get_default_value()))
+            lines.append('')
+        return '\n'.join(lines)
+    
+    
 
 class SingletonConfigurable(Configurable):
     """A configurable that only allows one instance.
@@ -168,7 +222,32 b' class SingletonConfigurable(Configurable):'
     """
 
     _instance = None
-
+    
+    @classmethod
+    def _walk_mro(cls):
+        """Walk the cls.mro() for parent classes that are also singletons
+        
+        For use in instance()
+        """
+        
+        for subclass in cls.mro():
+            if issubclass(cls, subclass) and \
+                    issubclass(subclass, SingletonConfigurable) and \
+                    subclass != SingletonConfigurable:
+                yield subclass
+    
+    @classmethod
+    def clear_instance(cls):
+        """unset _instance for this class and singleton parents.
+        """
+        if not cls.initialized():
+            return
+        for subclass in cls._walk_mro():
+            if isinstance(subclass._instance, cls):
+                # only clear instances that are instances
+                # of the calling class
+                subclass._instance = None
+            
     @classmethod
     def instance(cls, *args, **kwargs):
         """Returns a global instance of this class.
@@ -202,14 +281,10 b' class SingletonConfigurable(Configurable):'
         if cls._instance is None:
             inst = cls(*args, **kwargs)
             # Now make sure that the instance will also be returned by
-            # the subclasses instance attribute.
-            for subclass in cls.mro():
-                if issubclass(cls, subclass) and \
-                issubclass(subclass, SingletonConfigurable) and \
-                subclass != SingletonConfigurable:
-                    subclass._instance = inst
-                else:
-                    break
+            # parent classes' _instance attribute.
+            for subclass in cls._walk_mro():
+                subclass._instance = inst
+        
         if isinstance(cls._instance, cls):
             return cls._instance
         else:
@@ -223,3 +298,18 b' class SingletonConfigurable(Configurable):'
         """Has an instance been created?"""
         return hasattr(cls, "_instance") and cls._instance is not None
 
+
+class LoggingConfigurable(Configurable):
+    """A parent class for Configurables that log.
+    
+    Subclasses have a log trait, and the default behavior
+    is to get the logger from the currently running Application
+    via Application.instance().log.
+    """
+    
+    log = Instance('logging.Logger')
+    def _log_default(self):
+        from IPython.config.application import Application
+        return Application.instance().log
+        
+        

IPython/config/loader.py

@@ -4,10 +4,11 b' Authors'
 -------
 * Brian Granger
 * Fernando Perez
+* Min RK
 """
 
 #-----------------------------------------------------------------------------
-#  Copyright (C) 2008-2009  The IPython Development Team
+#  Copyright (C) 2008-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
@@ -22,7 +23,7 b' import re'
 import sys
 
 from IPython.external import argparse
-from IPython.utils.path import filefind
+from IPython.utils.path import filefind, get_ipython_dir
 
 #-----------------------------------------------------------------------------
 # Exceptions
@@ -36,6 +37,9 b' class ConfigError(Exception):'
 class ConfigLoaderError(ConfigError):
     pass
 
+class ArgumentError(ConfigLoaderError):
+    pass
+
 #-----------------------------------------------------------------------------
 # Argparse fix
 #-----------------------------------------------------------------------------
@@ -265,23 +269,40 b' class PyFileConfigLoader(FileConfigLoader):'
     def _read_file_as_dict(self):
         """Load the config file into self.config, with recursive loading."""
         # This closure is made available in the namespace that is used
-        # to exec the config file.  This allows users to call
+        # to exec the config file.  It allows users to call
         # load_subconfig('myconfig.py') to load config files recursively.
         # It needs to be a closure because it has references to self.path
         # and self.config.  The sub-config is loaded with the same path
         # as the parent, but it uses an empty config which is then merged
         # with the parents.
-        def load_subconfig(fname):
-            loader = PyFileConfigLoader(fname, self.path)
+        
+        # If a profile is specified, the config file will be loaded
+        # from that profile
+        
+        def load_subconfig(fname, profile=None):
+            # import here to prevent circular imports
+            from IPython.core.profiledir import ProfileDir, ProfileDirError
+            if profile is not None:
+                try:
+                    profile_dir = ProfileDir.find_profile_dir_by_name(
+                            get_ipython_dir(),
+                            profile,
+                    )
+                except ProfileDirError:
+                    return
+                path = profile_dir.location
+            else:
+                path = self.path
+            loader = PyFileConfigLoader(fname, path)
             try:
                 sub_config = loader.load_config()
             except IOError:
                 # Pass silently if the sub config is not there. This happens
-                # when a user us using a profile, but not the default config.
+                # when a user s using a profile, but not the default config.
                 pass
             else:
                 self.config._merge(sub_config)
-
+        
         # Again, this needs to be a closure and should be used in config
         # files to get the config being loaded.
         def get_config():
@@ -304,7 +325,7 b' class CommandLineConfigLoader(ConfigLoader):'
     here.
     """
 
-kv_pattern = re.compile(r'[A-Za-z]\w*(\.\w+)*\=.+')
+kv_pattern = re.compile(r'[A-Za-z]\w*(\.\w+)*\=.*')
 flag_pattern = re.compile(r'\-\-\w+(\-\w)*')
 
 class KeyValueConfigLoader(CommandLineConfigLoader):
@@ -346,15 +367,27 b' class KeyValueConfigLoader(CommandLineConfigLoader):'
             >>> cl.load_config(["foo='bar'","A.name='brian'","B.number=0"])
             {'A': {'name': 'brian'}, 'B': {'number': 0}, 'foo': 'bar'}
         """
+        self.clear()
         if argv is None:
             argv = sys.argv[1:]
         self.argv = argv
         self.aliases = aliases or {}
         self.flags = flags or {}
+        
+    
+    def clear(self):
+        super(KeyValueConfigLoader, self).clear()
+        self.extra_args = []
+        
 
     def load_config(self, argv=None, aliases=None, flags=None):
         """Parse the configuration and generate the Config object.
-
+        
+        After loading, any arguments that are not key-value or
+        flags will be stored in self.extra_args - a list of
+        unparsed command-line arguments.  This is used for
+        arguments such as input files or subcommands.
+        
         Parameters
         ----------
         argv : list, optional
@@ -379,7 +412,7 b' class KeyValueConfigLoader(CommandLineConfigLoader):'
             aliases = self.aliases
         if flags is None:
             flags = self.flags
-
+        
         for item in argv:
             if kv_pattern.match(item):
                 lhs,rhs = item.split('=',1)
@@ -403,14 +436,21 b' class KeyValueConfigLoader(CommandLineConfigLoader):'
                 m = item[2:]
                 cfg,_ = flags.get(m, (None,None))
                 if cfg is None:
-                    raise ValueError("Unrecognized flag: %r"%item)
+                    raise ArgumentError("Unrecognized flag: %r"%item)
                 elif isinstance(cfg, (dict, Config)):
-                    # update self.config with Config:
-                    self.config.update(cfg)
+                    # don't clobber whole config sections, update
+                    # each section from config:
+                    for sec,c in cfg.iteritems():
+                        self.config[sec].update(c)
                 else:
                     raise ValueError("Invalid flag: %r"%flag)
+            elif item.startswith('-'):
+                # this shouldn't ever be valid
+                raise ArgumentError("Invalid argument: %r"%item)
             else:
-                raise ValueError("Invalid argument: %r"%item)
+                # keep all args that aren't valid in a list, 
+                # in case our parent knows what to do with them.
+                self.extra_args.append(item)
         return self.config
 
 class ArgParseConfigLoader(CommandLineConfigLoader):

IPython/config/profile/cluster/ipython_config.py

@@ -1,24 +1,25 b''
 c = get_config()
+app = c.InteractiveShellApp
 
 # This can be used at any point in a config file to load a sub config
 # and merge it into the current one.
-load_subconfig('ipython_config.py')
+load_subconfig('ipython_config.py', profile='default')
 
 lines = """
-from IPython.kernel.client import *
+from IPython.parallel import *
 """
 
 # You have to make sure that attributes that are containers already
 # exist before using them.  Simple assigning a new list will override
 # all previous values.
-if hasattr(c.Global, 'exec_lines'):
-    c.Global.exec_lines.append(lines)
+if hasattr(app, 'exec_lines'):
+    app.exec_lines.append(lines)
 else:
-    c.Global.exec_lines = [lines]
+    app.exec_lines = [lines]
 
 # Load the parallelmagic extension to enable %result, %px, %autopx magics.
-if hasattr(c.Global, 'extensions'):
-    c.Global.extensions.append('parallelmagic')
+if hasattr(app, 'extensions'):
+    app.extensions.append('parallelmagic')
 else:
-    c.Global.extensions = ['parallelmagic']
+    app.extensions = ['parallelmagic']
 

IPython/config/profile/math/ipython_config.py

@@ -1,8 +1,9 b''
 c = get_config()
+app = c.InteractiveShellApp
 
 # This can be used at any point in a config file to load a sub config
 # and merge it into the current one.
-load_subconfig('ipython_config.py')
+load_subconfig('ipython_config.py', profile='default')
 
 lines = """
 import cmath
@@ -12,8 +13,9 b' from math import *'
 # You have to make sure that attributes that are containers already
 # exist before using them.  Simple assigning a new list will override
 # all previous values.
-if hasattr(c.Global, 'exec_lines'):
-    c.Global.exec_lines.append(lines)
+
+if hasattr(app, 'exec_lines'):
+    app.exec_lines.append(lines)
 else:
-    c.Global.exec_lines = [lines]
+    app.exec_lines = [lines]
 

IPython/config/profile/pylab/ipython_config.py

@@ -1,13 +1,14 b''
 c = get_config()
+app = c.InteractiveShellApp
 
 # This can be used at any point in a config file to load a sub config
 # and merge it into the current one.
-load_subconfig('ipython_config.py')
+load_subconfig('ipython_config.py', profile='default')
 
 lines = """
 import matplotlib
-%gui -a wx
-matplotlib.use('wxagg')
+%gui qt
+matplotlib.use('qtagg')
 matplotlib.interactive(True)
 from matplotlib import pyplot as plt
 from matplotlib.pyplot import *
@@ -16,7 +17,7 b' from matplotlib.pyplot import *'
 # You have to make sure that attributes that are containers already
 # exist before using them.  Simple assigning a new list will override
 # all previous values.
-if hasattr(c.Global, 'exec_lines'):
-    c.Global.exec_lines.append(lines)
+if hasattr(app, 'exec_lines'):
+    app.exec_lines.append(lines)
 else:
-    c.Global.exec_lines = [lines]
  No newline at end of file
+    app.exec_lines = [lines]
  No newline at end of file

IPython/config/profile/pysh/ipython_config.py

@@ -1,8 +1,9 b''
 c = get_config()
+app = c.InteractiveShellApp
 
 # This can be used at any point in a config file to load a sub config
 # and merge it into the current one.
-load_subconfig('ipython_config.py')
+load_subconfig('ipython_config.py', profile='default')
 
 c.InteractiveShell.prompt_in1 = '\C_LightGreen\u@\h\C_LightBlue[\C_LightCyan\Y1\C_LightBlue]\C_Green|\#> '
 c.InteractiveShell.prompt_in2 = '\C_Green|\C_LightGreen\D\C_Green> '
@@ -23,7 +24,7 b' lines = """'
 # You have to make sure that attributes that are containers already
 # exist before using them.  Simple assigning a new list will override
 # all previous values.
-if hasattr(c.Global, 'exec_lines'):
-    c.Global.exec_lines.append(lines)
+if hasattr(app, 'exec_lines'):
+    app.exec_lines.append(lines)
 else:
-    c.Global.exec_lines = [lines]
  No newline at end of file
+    app.exec_lines = [lines]
  No newline at end of file

IPython/config/profile/sympy/ipython_config.py

@@ -1,8 +1,9 b''
 c = get_config()
+app = c.InteractiveShellApp
 
 # This can be used at any point in a config file to load a sub config
 # and merge it into the current one.
-load_subconfig('ipython_config.py')
+load_subconfig('ipython_config.py', profile='default')
 
 lines = """
 from __future__ import division
@@ -16,14 +17,14 b" f, g, h = map(Function, 'fgh')"
 # exist before using them.  Simple assigning a new list will override
 # all previous values.
 
-if hasattr(c.Global, 'exec_lines'):
-    c.Global.exec_lines.append(lines)
+if hasattr(app, 'exec_lines'):
+    app.exec_lines.append(lines)
 else:
-    c.Global.exec_lines = [lines]
+    app.exec_lines = [lines]
 
 # Load the sympy_printing extension to enable nice printing of sympy expr's.
-if hasattr(c.Global, 'extensions'):
-    c.Global.extensions.append('sympy_printing')
+if hasattr(app, 'extensions'):
+    app.extensions.append('sympyprinting')
 else:
-    c.Global.extensions = ['sympy_printing']
+    app.extensions = ['sympyprinting']
 

IPython/config/tests/test_application.py

@@ -42,12 +42,13 b' class Foo(Configurable):'
 
 class Bar(Configurable):
 
+    b = Int(0, config=True, help="The integer b.")
     enabled = Bool(True, config=True, help="Enable bar.")
 
 
 class MyApp(Application):
 
-    app_name = Unicode(u'myapp')
+    name = Unicode(u'myapp')
     running = Bool(False, config=True,
                    help="Is the app running?")
     classes = List([Bar, Foo])
@@ -71,30 +72,30 b' class TestApplication(TestCase):'
 
     def test_basic(self):
         app = MyApp()
-        self.assertEquals(app.app_name, u'myapp')
+        self.assertEquals(app.name, u'myapp')
         self.assertEquals(app.running, False)
         self.assertEquals(app.classes, [MyApp,Bar,Foo])
         self.assertEquals(app.config_file, u'')
 
     def test_config(self):
         app = MyApp()
-        app.parse_command_line(["i=10","Foo.j=10","enabled=False","log_level=0"])
+        app.parse_command_line(["i=10","Foo.j=10","enabled=False","log_level=50"])
         config = app.config
         self.assertEquals(config.Foo.i, 10)
         self.assertEquals(config.Foo.j, 10)
         self.assertEquals(config.Bar.enabled, False)
-        self.assertEquals(config.MyApp.log_level,0)
+        self.assertEquals(config.MyApp.log_level,50)
 
     def test_config_propagation(self):
         app = MyApp()
-        app.parse_command_line(["i=10","Foo.j=10","enabled=False","log_level=0"])
+        app.parse_command_line(["i=10","Foo.j=10","enabled=False","log_level=50"])
         app.init_foo()
         app.init_bar()
         self.assertEquals(app.foo.i, 10)
         self.assertEquals(app.foo.j, 10)
         self.assertEquals(app.bar.enabled, False)
 
-    def test_alias(self):
+    def test_flags(self):
         app = MyApp()
         app.parse_command_line(["--disable"])
         app.init_bar()
@@ -103,3 +104,32 b' class TestApplication(TestCase):'
         app.init_bar()
         self.assertEquals(app.bar.enabled, True)
     
+    def test_aliases(self):
+        app = MyApp()
+        app.parse_command_line(["i=5", "j=10"])
+        app.init_foo()
+        self.assertEquals(app.foo.i, 5)
+        app.init_foo()
+        self.assertEquals(app.foo.j, 10)
+    
+    def test_flag_clobber(self):
+        """test that setting flags doesn't clobber existing settings"""
+        app = MyApp()
+        app.parse_command_line(["Bar.b=5", "--disable"])
+        app.init_bar()
+        self.assertEquals(app.bar.enabled, False)
+        self.assertEquals(app.bar.b, 5)
+        app.parse_command_line(["--enable", "Bar.b=10"])
+        app.init_bar()
+        self.assertEquals(app.bar.enabled, True)
+        self.assertEquals(app.bar.b, 10)
+    
+    def test_extra_args(self):
+        app = MyApp()
+        app.parse_command_line(['extra', "Bar.b=5", "--disable", 'args'])
+        app.init_bar()
+        self.assertEquals(app.bar.enabled, False)
+        self.assertEquals(app.bar.b, 5)
+        self.assertEquals(app.extra_args, ['extra', 'args'])
+    
+

IPython/config/tests/test_configurable.py

@@ -48,8 +48,10 b' class MyConfigurable(Configurable):'
 mc_help=u"""MyConfigurable options
 ----------------------
 MyConfigurable.a : Int
+    Default: 1
     The integer a.
 MyConfigurable.b : Float
+    Default: 1.0
     The integer b."""
 
 class Foo(Configurable):

IPython/config/tests/test_loader.py

@@ -123,6 +123,13 b' class TestKeyValueCL(TestCase):'
         self.assertEquals(config.Foo.Bar.value, 10)
         self.assertEquals(config.Foo.Bam.value, range(10))
         self.assertEquals(config.D.C.value, 'hi there')
+    
+    def test_extra_args(self):
+        cl = KeyValueConfigLoader()
+        config = cl.load_config(['a=5', 'b', 'c=10', 'd'])
+        self.assertEquals(cl.extra_args, ['b', 'd'])
+        self.assertEquals(config.a, 5)
+        self.assertEquals(config.c, 10)
 
 
 class TestConfig(TestCase):

IPython/core/application.py

@@ -12,13 +12,12 b' Authors:'
 
 * Brian Granger
 * Fernando Perez
+* Min RK
 
-Notes
------
 """
 
 #-----------------------------------------------------------------------------
-#  Copyright (C) 2008-2009  The IPython Development Team
+#  Copyright (C) 2008-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
@@ -30,439 +29,262 b' Notes'
 
 import logging
 import os
+import shutil
 import sys
 
+from IPython.config.application import Application
+from IPython.config.configurable import Configurable
+from IPython.config.loader import Config
 from IPython.core import release, crashhandler
+from IPython.core.profiledir import ProfileDir, ProfileDirError
 from IPython.utils.path import get_ipython_dir, get_ipython_package_dir
-from IPython.config.loader import (
-    PyFileConfigLoader,
-    ArgParseConfigLoader,
-    Config,
-)
+from IPython.utils.traitlets import List, Unicode, Type, Bool, Dict
 
 #-----------------------------------------------------------------------------
 # Classes and functions
 #-----------------------------------------------------------------------------
 
-class ApplicationError(Exception):
-    pass
-
-
-class BaseAppConfigLoader(ArgParseConfigLoader):
-    """Default command line options for IPython based applications."""
-
-    def _add_ipython_dir(self, parser):
-        """Add the --ipython-dir option to the parser."""
-        paa = parser.add_argument
-        paa('--ipython-dir', 
-            dest='Global.ipython_dir',type=unicode,
-            help=
-            """Set to override default location of the IPython directory
-            IPYTHON_DIR, stored as Global.ipython_dir.  This can also be 
-            specified through the environment variable IPYTHON_DIR.""",
-            metavar='Global.ipython_dir')
-
-    def _add_log_level(self, parser):
-        """Add the --log-level option to the parser."""
-        paa = parser.add_argument
-        paa('--log-level',
-            dest="Global.log_level",type=int,
-            help='Set the log level (0,10,20,30,40,50).  Default is 30.',
-            metavar='Global.log_level')
-            
-    def _add_version(self, parser):
-        """Add the --version option to the parser."""
-        parser.add_argument('--version', action="version",
-                                                version=self.version)
-
-    def _add_arguments(self):
-        self._add_ipython_dir(self.parser)
-        self._add_log_level(self.parser)
-        try:  # Old versions of argparse don't have a version action
-            self._add_version(self.parser)
-        except Exception:
-            pass
-
-
-class Application(object):
-    """Load a config, construct configurables and set them running.
-
-    The configuration of an application can be done via three different Config
-    objects, which are loaded and ultimately merged into a single one used
-    from that point on by the app. These are:
-
-    1. default_config: internal defaults, implemented in code.
-    2. file_config: read from the filesystem.
-    3. command_line_config: read from the system's command line flags.
-
-    During initialization, 3 is actually read before 2, since at the
-    command-line one may override the location of the file to be read.  But the
-    above is the order in which the merge is made.
-    """
-
-    name = u'ipython'
-    description = 'IPython: an enhanced interactive Python shell.'
-    #: Usage message printed by argparse. If None, auto-generate
-    usage = None
-    #: The command line config loader.  Subclass of ArgParseConfigLoader.
-    command_line_loader = BaseAppConfigLoader
-    #: The name of the config file to load, determined at runtime
-    config_file_name = None
-    #: The name of the default config file. Track separately from the actual
-    #: name because some logic happens only if we aren't using the default.
-    default_config_file_name = u'ipython_config.py'
-    default_log_level = logging.WARN
-    #: Set by --profile option
-    profile_name = None
-    #: User's ipython directory, typically ~/.ipython or ~/.config/ipython/
-    ipython_dir = None
-    #: Internal defaults, implemented in code.
-    default_config = None
-    #: Read from the filesystem.
-    file_config = None
-    #: Read from the system's command line flags.
-    command_line_config = None
-    #: The final config that will be passed to the main object.
-    master_config = None
-    #: A reference to the argv to be used (typically ends up being sys.argv[1:])
-    argv = None
-    #: extra arguments computed by the command-line loader
-    extra_args = None
-    #: The class to use as the crash handler.
-    crash_handler_class = crashhandler.CrashHandler
-
-    # Private attributes
-    _exiting = False
-    _initialized = False
-
-    def __init__(self, argv=None):
-        self.argv = sys.argv[1:] if argv is None else argv
-        self.init_logger()
-
-    def init_logger(self):
-        self.log = logging.getLogger(self.__class__.__name__)
-        # This is used as the default until the command line arguments are read.
-        self.log.setLevel(self.default_log_level)
-        self._log_handler = logging.StreamHandler()
-        self._log_formatter = logging.Formatter("[%(name)s] %(message)s")
-        self._log_handler.setFormatter(self._log_formatter)
-        self.log.addHandler(self._log_handler)
-
-    def _set_log_level(self, level):
-        self.log.setLevel(level)
-
-    def _get_log_level(self):
-        return self.log.level
-
-    log_level = property(_get_log_level, _set_log_level)
-
-    def initialize(self):
-        """Initialize the application.
-
-        Loads all configuration information and sets all application state, but
-        does not start any relevant processing (typically some kind of event
-        loop).
-
-        Once this method has been called, the application is flagged as
-        initialized and the method becomes a no-op."""
-        
-        if self._initialized:
-            return
-
-        # The first part is protected with an 'attempt' wrapper, that will log
-        # failures with the basic system traceback machinery.  Once our crash
-        # handler is in place, we can let any subsequent exception propagate,
-        # as our handler will log it with much better detail than the default.
-        self.attempt(self.create_crash_handler)
-
-        # Configuration phase
-        # Default config (internally hardwired in application code)
-        self.create_default_config()
-        self.log_default_config()
-        self.set_default_config_log_level()
-
-        # Command-line config
-        self.pre_load_command_line_config()
-        self.load_command_line_config()
-        self.set_command_line_config_log_level()
-        self.post_load_command_line_config()
-        self.log_command_line_config()
 
-        # Find resources needed for filesystem access, using information from
-        # the above two
-        self.find_ipython_dir()
-        self.find_resources()
-        self.find_config_file_name()
-        self.find_config_file_paths()
-
-        # File-based config
-        self.pre_load_file_config()
-        self.load_file_config()
-        self.set_file_config_log_level()
-        self.post_load_file_config()
-        self.log_file_config()
+#-----------------------------------------------------------------------------
+# Base Application Class
+#-----------------------------------------------------------------------------
 
-        # Merge all config objects into a single one the app can then use
-        self.merge_configs()
-        self.log_master_config()
+# aliases and flags
 
-        # Construction phase
-        self.pre_construct()
-        self.construct()
-        self.post_construct()
+base_aliases = dict(
+    profile='BaseIPythonApplication.profile',
+    ipython_dir='BaseIPythonApplication.ipython_dir',
+)
 
-        # Done, flag as such and
-        self._initialized = True
+base_flags = dict(
+    debug = ({'Application' : {'log_level' : logging.DEBUG}},
+            "set log level to logging.DEBUG (maximize logging output)"),
+    quiet = ({'Application' : {'log_level' : logging.CRITICAL}},
+            "set log level to logging.CRITICAL (minimize logging output)"),
+    init = ({'BaseIPythonApplication' : {
+                    'copy_config_files' : True,
+                    'auto_create' : True}
+            }, "Initialize profile with default config files")
+)
 
-    def start(self):
-        """Start the application."""
-        self.initialize()
-        self.start_app()
 
+class BaseIPythonApplication(Application):
+
+    name = Unicode(u'ipython')
+    description = Unicode(u'IPython: an enhanced interactive Python shell.')
+    version = Unicode(release.version)
+    
+    aliases = Dict(base_aliases)
+    flags = Dict(base_flags)
+    
+    # Track whether the config_file has changed,
+    # because some logic happens only if we aren't using the default.
+    config_file_specified = Bool(False)
+    
+    config_file_name = Unicode(u'ipython_config.py')
+    def _config_file_name_default(self):
+        return self.name.replace('-','_') + u'_config.py'
+    def _config_file_name_changed(self, name, old, new):
+        if new != old:
+            self.config_file_specified = True
+
+    # The directory that contains IPython's builtin profiles.
+    builtin_profile_dir = Unicode(
+        os.path.join(get_ipython_package_dir(), u'config', u'profile', u'default')
+    )
+
+    config_file_paths = List(Unicode)
+    def _config_file_paths_default(self):
+        return [os.getcwdu()]
+
+    profile = Unicode(u'default', config=True,
+        help="""The IPython profile to use."""
+    )
+    def _profile_changed(self, name, old, new):
+        self.builtin_profile_dir = os.path.join(
+                get_ipython_package_dir(), u'config', u'profile', new
+        )
+    
+    ipython_dir = Unicode(get_ipython_dir(), config=True, 
+        help="""
+        The name of the IPython directory. This directory is used for logging
+        configuration (through profiles), history storage, etc. The default
+        is usually $HOME/.ipython. This options can also be specified through
+        the environment variable IPYTHON_DIR.
+        """
+    )
+    
+    overwrite = Bool(False, config=True,
+        help="""Whether to overwrite existing config files when copying""")
+    auto_create = Bool(False, config=True,
+        help="""Whether to create profile dir if it doesn't exist""")
+    
+    config_files = List(Unicode)
+    def _config_files_default(self):
+        return [u'ipython_config.py']
+    
+    copy_config_files = Bool(False, config=True,
+        help="""Whether to install the default config files into the profile dir.
+        If a new profile is being created, and IPython contains config files for that
+        profile, then they will be staged into the new directory.  Otherwise,
+        default config files will be automatically generated.
+        """)
+
+    # The class to use as the crash handler.
+    crash_handler_class = Type(crashhandler.CrashHandler)
+
+    def __init__(self, **kwargs):
+        super(BaseIPythonApplication, self).__init__(**kwargs)
+        # ensure even default IPYTHON_DIR exists
+        if not os.path.exists(self.ipython_dir):
+            self._ipython_dir_changed('ipython_dir', self.ipython_dir, self.ipython_dir)
+    
     #-------------------------------------------------------------------------
     # Various stages of Application creation
     #-------------------------------------------------------------------------
 
-    def create_crash_handler(self):
+    def init_crash_handler(self):
         """Create a crash handler, typically setting sys.excepthook to it."""
         self.crash_handler = self.crash_handler_class(self)
         sys.excepthook = self.crash_handler
 
-    def create_default_config(self):
-        """Create defaults that can't be set elsewhere.
-
-        For the most part, we try to set default in the class attributes
-        of Configurables.  But, defaults the top-level Application (which is
-        not a HasTraits or Configurables) are not set in this way.  Instead
-        we set them here.  The Global section is for variables like this that
-        don't belong to a particular configurable.
-        """
-        c = Config()
-        c.Global.ipython_dir = get_ipython_dir()
-        c.Global.log_level = self.log_level
-        self.default_config = c
-
-    def log_default_config(self):
-        self.log.debug('Default config loaded:')
-        self.log.debug(repr(self.default_config))
-
-    def set_default_config_log_level(self):
-        try:
-            self.log_level = self.default_config.Global.log_level
-        except AttributeError:
-            # Fallback to the default_log_level class attribute
-            pass
-
-    def create_command_line_config(self):
-        """Create and return a command line config loader."""
-        return self.command_line_loader(
-            self.argv,
-            description=self.description, 
-            version=release.version,
-            usage=self.usage
-        )
-
-    def pre_load_command_line_config(self):
-        """Do actions just before loading the command line config."""
-        pass
-
-    def load_command_line_config(self):
-        """Load the command line config."""
-        loader = self.create_command_line_config()
-        self.command_line_config = loader.load_config()
-        self.extra_args = loader.get_extra_args()
-
-    def set_command_line_config_log_level(self):
-        try:
-            self.log_level = self.command_line_config.Global.log_level
-        except AttributeError:
-            pass
-
-    def post_load_command_line_config(self):
-        """Do actions just after loading the command line config."""
-        pass
-
-    def log_command_line_config(self):
-        self.log.debug("Command line config loaded:")
-        self.log.debug(repr(self.command_line_config))
-
-    def find_ipython_dir(self):
-        """Set the IPython directory.
-
-        This sets ``self.ipython_dir``, but the actual value that is passed to
-        the application is kept in either ``self.default_config`` or
-        ``self.command_line_config``.  This also adds ``self.ipython_dir`` to
-        ``sys.path`` so config files there can be referenced by other config
-        files.
-        """
-
-        try:
-            self.ipython_dir = self.command_line_config.Global.ipython_dir
-        except AttributeError:
-            self.ipython_dir = self.default_config.Global.ipython_dir
-        sys.path.append(os.path.abspath(self.ipython_dir))
-        if not os.path.isdir(self.ipython_dir):
-            os.makedirs(self.ipython_dir, mode=0777)
-        self.log.debug("IPYTHON_DIR set to: %s" % self.ipython_dir)
-
-    def find_resources(self):
-        """Find other resources that need to be in place.
-
-        Things like cluster directories need to be in place to find the
-        config file.  These happen right after the IPython directory has
-        been set.
-        """
-        pass
-
-    def find_config_file_name(self):
-        """Find the config file name for this application.
-
-        This must set ``self.config_file_name`` to the filename of the
-        config file to use (just the filename). The search paths for the
-        config file are set in :meth:`find_config_file_paths` and then passed
-        to the config file loader where they are resolved to an absolute path.
-
-        If a profile has been set at the command line, this will resolve it.
-        """
-        try:
-            self.config_file_name = self.command_line_config.Global.config_file
-        except AttributeError:
-            pass
-        else:
-            return
-
-        try:
-            self.profile_name = self.command_line_config.Global.profile
-        except AttributeError:
-            # Just use the default as there is no profile
-            self.config_file_name = self.default_config_file_name
-        else:
-            # Use the default config file name and profile name if set
-            # to determine the used config file name.
-            name_parts = self.default_config_file_name.split('.')
-            name_parts.insert(1, u'_' + self.profile_name + u'.')
-            self.config_file_name = ''.join(name_parts)
-
-    def find_config_file_paths(self):
-        """Set the search paths for resolving the config file.
-
-        This must set ``self.config_file_paths`` to a sequence of search
-        paths to pass to the config file loader.
-        """
-        # Include our own profiles directory last, so that users can still find
-        # our shipped copies of builtin profiles even if they don't have them
-        # in their local ipython directory.
-        prof_dir = os.path.join(get_ipython_package_dir(), 'config', 'profile')
-        self.config_file_paths = (os.getcwdu(), self.ipython_dir, prof_dir)
-
-    def pre_load_file_config(self):
-        """Do actions before the config file is loaded."""
-        pass
-
-    def load_file_config(self, suppress_errors=True):
+    def _ipython_dir_changed(self, name, old, new):
+        if old in sys.path:
+            sys.path.remove(old)
+        sys.path.append(os.path.abspath(new))
+        if not os.path.isdir(new):
+            os.makedirs(new, mode=0777)
+        readme = os.path.join(new, 'README')
+        if not os.path.exists(readme):
+            path = os.path.join(get_ipython_package_dir(), u'config', u'profile')
+            shutil.copy(os.path.join(path, 'README'), readme)
+        self.log.debug("IPYTHON_DIR set to: %s" % new)
+
+    def load_config_file(self, suppress_errors=True):
         """Load the config file.
-        
-        This tries to load the config file from disk.  If successful, the
-        ``CONFIG_FILE`` config variable is set to the resolved config file
-        location.  If not successful, an empty config is used.
-        
+
         By default, errors in loading config are handled, and a warning
         printed on screen. For testing, the suppress_errors option is set
         to False, so errors will make tests fail.
         """
+        base_config = 'ipython_config.py'
+        self.log.debug("Attempting to load config file: %s" %
+                       base_config)
+        try:
+            Application.load_config_file(
+                self,
+                base_config, 
+                path=self.config_file_paths
+            )
+        except IOError:
+            # ignore errors loading parent
+            pass
+        if self.config_file_name == base_config:
+            # don't load secondary config
+            return
         self.log.debug("Attempting to load config file: %s" %
                        self.config_file_name)
-        loader = PyFileConfigLoader(self.config_file_name,
-                                    path=self.config_file_paths)
         try:
-            self.file_config = loader.load_config()
-            self.file_config.Global.config_file = loader.full_filename
+            Application.load_config_file(
+                self,
+                self.config_file_name, 
+                path=self.config_file_paths
+            )
         except IOError:
             # Only warn if the default config file was NOT being used.
-            if not self.config_file_name==self.default_config_file_name:
+            if self.config_file_specified:
                 self.log.warn("Config file not found, skipping: %s" %
-                               self.config_file_name, exc_info=True)
-            self.file_config = Config()
+                               self.config_file_name)
         except:
-            if not suppress_errors:     # For testing purposes
+            # For testing purposes.
+            if not suppress_errors:
                 raise
             self.log.warn("Error loading config file: %s" %
                           self.config_file_name, exc_info=True)
-            self.file_config = Config()
 
-    def set_file_config_log_level(self):
-        # We need to keeep self.log_level updated.  But we only use the value
-        # of the file_config if a value was not specified at the command
-        # line, because the command line overrides everything.
-        if not hasattr(self.command_line_config.Global, 'log_level'):
+    def init_profile_dir(self):
+        """initialize the profile dir"""
+        try:
+            # location explicitly specified:
+            location = self.config.ProfileDir.location
+        except AttributeError:
+            # location not specified, find by profile name
             try:
-                self.log_level = self.file_config.Global.log_level
-            except AttributeError:
-                pass # Use existing value
-
-    def post_load_file_config(self):
-        """Do actions after the config file is loaded."""
-        pass
-
-    def log_file_config(self):
-        if hasattr(self.file_config.Global, 'config_file'):
-            self.log.debug("Config file loaded: %s" %
-                           self.file_config.Global.config_file)
-            self.log.debug(repr(self.file_config))
-
-    def merge_configs(self):
-        """Merge the default, command line and file config objects."""
-        config = Config()
-        config._merge(self.default_config)
-        config._merge(self.file_config)
-        config._merge(self.command_line_config)
-
-        # XXX fperez - propose to Brian we rename master_config to simply
-        # config, I think this is going to be heavily used in examples and
-        # application code and the name is shorter/easier to find/remember.
-        # For now, just alias it...
-        self.master_config = config
-        self.config = config
-
-    def log_master_config(self):
-        self.log.debug("Master config created:")
-        self.log.debug(repr(self.master_config))
-
-    def pre_construct(self):
-        """Do actions after the config has been built, but before construct."""
-        pass
-
-    def construct(self):
-        """Construct the main objects that make up this app."""
-        self.log.debug("Constructing main objects for application")
-
-    def post_construct(self):
-        """Do actions after construct, but before starting the app."""
-        pass
-
-    def start_app(self):
-        """Actually start the app."""
-        self.log.debug("Starting application")
-
-    #-------------------------------------------------------------------------
-    # Utility methods
-    #-------------------------------------------------------------------------
-
-    def exit(self, exit_status=0):
-        if self._exiting:
-            pass
+                p = ProfileDir.find_profile_dir_by_name(self.ipython_dir, self.profile, self.config)
+            except ProfileDirError:
+                # not found, maybe create it (always create default profile)
+                if self.auto_create or self.profile=='default':
+                    try:
+                        p = ProfileDir.create_profile_dir_by_name(self.ipython_dir, self.profile, self.config)
+                    except ProfileDirError:
+                        self.log.fatal("Could not create profile: %r"%self.profile)
+                        self.exit(1)
+                    else:
+                        self.log.info("Created profile dir: %r"%p.location)
+                else:
+                    self.log.fatal("Profile %r not found."%self.profile)
+                    self.exit(1)
+            else:
+                self.log.info("Using existing profile dir: %r"%p.location)
         else:
-            self.log.debug("Exiting application: %s" % self.name)
-            self._exiting = True
-            sys.exit(exit_status)
-
-    def attempt(self, func):
-        try:
-            func()
-        except SystemExit:
-            raise
-        except:
-            self.log.critical("Aborting application: %s" % self.name,
-                              exc_info=True)
-            self.exit(0)
+            # location is fully specified
+            try:
+                p = ProfileDir.find_profile_dir(location, self.config)
+            except ProfileDirError:
+                # not found, maybe create it
+                if self.auto_create:
+                    try:
+                        p = ProfileDir.create_profile_dir(location, self.config)
+                    except ProfileDirError:
+                        self.log.fatal("Could not create profile directory: %r"%location)
+                        self.exit(1)
+                    else:
+                        self.log.info("Creating new profile dir: %r"%location)
+                else:
+                    self.log.fatal("Profile directory %r not found."%location)
+                    self.exit(1)
+            else:
+                self.log.info("Using existing profile dir: %r"%location)
+        
+        self.profile_dir = p
+        self.config_file_paths.append(p.location)
+    
+    def init_config_files(self):
+        """[optionally] copy default config files into profile dir."""
+        # copy config files
+        if self.copy_config_files:
+            path = self.builtin_profile_dir
+            src = self.profile
+            
+            cfg = self.config_file_name
+            if path and os.path.exists(os.path.join(path, cfg)):
+                self.log.warn("Staging %r from %s into %r [overwrite=%s]"%(
+                        cfg, src, self.profile_dir.location, self.overwrite)
+                )
+                self.profile_dir.copy_config_file(cfg, path=path, overwrite=self.overwrite)
+            else:
+                self.stage_default_config_file()
+    
+    def stage_default_config_file(self):
+        """auto generate default config file, and stage it into the profile."""
+        s = self.generate_config_file()
+        fname = os.path.join(self.profile_dir.location, self.config_file_name)
+        if self.overwrite or not os.path.exists(fname):
+            self.log.warn("Generating default config file: %r"%(fname))
+            with open(fname, 'w') as f:
+                f.write(s)
+        
+    
+    def initialize(self, argv=None):
+        self.init_crash_handler()
+        self.parse_command_line(argv)
+        if self.subapp is not None:
+            # stop here if subapp is taking over
+            return
+        cl_config = self.config
+        self.init_profile_dir()
+        self.init_config_files()
+        self.load_config_file()
+        # enforce cl-opts override configfile opts:
+        self.update_config(cl_config)
 

IPython/core/history.py

@@ -101,18 +101,15 b' class HistoryManager(Configurable):'
 
         if self.hist_file == u'':
             # No one has set the hist_file, yet.
-            if shell.profile:
-                histfname = 'history-%s' % shell.profile
-            else:
-                histfname = 'history'
-            self.hist_file = os.path.join(shell.ipython_dir, histfname + '.sqlite')
+            histfname = 'history'
+            self.hist_file = os.path.join(shell.profile_dir.location, histfname + '.sqlite')
 
         try:
             self.init_db()
         except sqlite3.DatabaseError:
             if os.path.isfile(self.hist_file):
                 # Try to move the file out of the way.
-                newpath = os.path.join(self.shell.ipython_dir, "hist-corrupt.sqlite")
+                newpath = os.path.join(self.shell.profile_dir.location, "hist-corrupt.sqlite")
                 os.rename(self.hist_file, newpath)
                 print("ERROR! History file wasn't a valid SQLite database.",
                 "It was moved to %s" % newpath, "and a new file created.")

IPython/core/interactiveshell.py

@@ -57,6 +57,7 b' from IPython.core.magic import Magic'
 from IPython.core.payload import PayloadManager
 from IPython.core.plugin import PluginManager
 from IPython.core.prefilter import PrefilterManager, ESC_MAGIC
+from IPython.core.profiledir import ProfileDir
 from IPython.external.Itpl import ItplNS
 from IPython.utils import PyColorize
 from IPython.utils import io
@@ -238,7 +239,9 b' class InteractiveShell(SingletonConfigurable, Magic):'
         """
     )
     colors = CaselessStrEnum(('NoColor','LightBG','Linux'), 
-                             default_value=get_default_colors(), config=True)
+                             default_value=get_default_colors(), config=True,
+        help="Set the color scheme (NoColor, Linux, or LightBG)."
+    )
     debug = CBool(False, config=True)
     deep_reload = CBool(False, config=True, help=
         """
@@ -291,7 +294,6 b' class InteractiveShell(SingletonConfigurable, Magic):'
         """
     )
 
-    profile = Unicode('', config=True)
     prompt_in1 = Str('In [\\#]: ', config=True)
     prompt_in2 = Str('   .\\D.: ', config=True)
     prompt_out = Str('Out[\\#]: ', config=True)
@@ -342,10 +344,18 b' class InteractiveShell(SingletonConfigurable, Magic):'
     payload_manager = Instance('IPython.core.payload.PayloadManager')
     history_manager = Instance('IPython.core.history.HistoryManager')
 
+    profile_dir = Instance('IPython.core.application.ProfileDir')
+    @property
+    def profile(self):
+        if self.profile_dir is not None:
+            name = os.path.basename(self.profile_dir.location)
+            return name.replace('profile_','')
+
+
     # Private interface
     _post_execute = Instance(dict)
 
-    def __init__(self, config=None, ipython_dir=None,
+    def __init__(self, config=None, ipython_dir=None, profile_dir=None,
                  user_ns=None, user_global_ns=None,
                  custom_exceptions=((), None)):
 
@@ -355,6 +365,7 b' class InteractiveShell(SingletonConfigurable, Magic):'
 
         # These are relatively independent and stateless
         self.init_ipython_dir(ipython_dir)
+        self.init_profile_dir(profile_dir)
         self.init_instance_attrs()
         self.init_environment()
 
@@ -372,7 +383,7 b' class InteractiveShell(SingletonConfigurable, Magic):'
         # While we're trying to have each part of the code directly access what
         # it needs without keeping redundant references to objects, we have too
         # much legacy code that expects ip.db to exist.
-        self.db = PickleShareDB(os.path.join(self.ipython_dir, 'db'))
+        self.db = PickleShareDB(os.path.join(self.profile_dir.location, 'db'))
 
         self.init_history()
         self.init_encoding()
@@ -457,16 +468,16 b' class InteractiveShell(SingletonConfigurable, Magic):'
     def init_ipython_dir(self, ipython_dir):
         if ipython_dir is not None:
             self.ipython_dir = ipython_dir
-            self.config.Global.ipython_dir = self.ipython_dir
             return
 
-        if hasattr(self.config.Global, 'ipython_dir'):
-            self.ipython_dir = self.config.Global.ipython_dir
-        else:
-            self.ipython_dir = get_ipython_dir()
+        self.ipython_dir = get_ipython_dir()
 
-        # All children can just read this
-        self.config.Global.ipython_dir = self.ipython_dir
+    def init_profile_dir(self, profile_dir):
+        if profile_dir is not None:
+            self.profile_dir = profile_dir
+            return
+        self.profile_dir =\
+            ProfileDir.create_profile_dir_by_name(self.ipython_dir, 'default')
 
     def init_instance_attrs(self):
         self.more = False

IPython/core/magic.py

@@ -46,6 +46,7 b' from IPython.core import debugger, oinspect'
 from IPython.core.error import TryNext
 from IPython.core.error import UsageError
 from IPython.core.fakemodule import FakeModule
+from IPython.core.profiledir import ProfileDir
 from IPython.core.macro import Macro
 from IPython.core import page
 from IPython.core.prefilter import ESC_MAGIC
@@ -533,10 +534,7 b' Currently the magic system has the following functions:\\n"""'
 
     def magic_profile(self, parameter_s=''):
         """Print your currently active IPython profile."""
-        if self.shell.profile:
-            printpl('Current IPython profile: $self.shell.profile.')
-        else:
-            print 'No profile active.'
+        print self.shell.profile
 
     def magic_pinfo(self, parameter_s='', namespaces=None):
         """Provide detailed information about an object.
@@ -3373,22 +3371,16 b' Defaulting color scheme to \'NoColor\'"""'
         else:
             overwrite = False
         from IPython.config import profile
-        profile_dir = os.path.split(profile.__file__)[0]
+        profile_dir = os.path.dirname(profile.__file__)
         ipython_dir = self.ipython_dir
-        files = os.listdir(profile_dir)
-
-        to_install = []
-        for f in files:
-            if f.startswith('ipython_config'):
-                src = os.path.join(profile_dir, f)
-                dst = os.path.join(ipython_dir, f)
-                if (not os.path.isfile(dst)) or overwrite:
-                    to_install.append((f, src, dst))
-        if len(to_install)>0:
-            print "Installing profiles to: ", ipython_dir
-            for (f, src, dst) in to_install:
-                shutil.copy(src, dst)
-                print "    %s" % f
+        print "Installing profiles to: %s [overwrite=%s]"%(ipython_dir,overwrite)
+        for src in os.listdir(profile_dir):
+            if src.startswith('profile_'):
+                name = src.replace('profile_', '')
+                print "    %s"%name
+                pd = ProfileDir.create_profile_dir_by_name(ipython_dir, name)
+                pd.copy_config_file('ipython_config.py', path=src, 
+                                overwrite=overwrite)
 
     @skip_doctest
     def magic_install_default_config(self, s):
@@ -3404,15 +3396,9 b' Defaulting color scheme to \'NoColor\'"""'
             overwrite = True
         else:
             overwrite = False
-        from IPython.config import default
-        config_dir = os.path.split(default.__file__)[0]
-        ipython_dir = self.ipython_dir
-        default_config_file_name = 'ipython_config.py'
-        src = os.path.join(config_dir, default_config_file_name)
-        dst = os.path.join(ipython_dir, default_config_file_name)
-        if (not os.path.isfile(dst)) or overwrite:
-            shutil.copy(src, dst)
-            print "Installing default config file: %s" % dst
+        pd = self.shell.profile_dir
+        print "Installing default config file in: %s" % pd.location
+        pd.copy_config_file('ipython_config.py', overwrite=overwrite)
 
     # Pylab support: simple wrappers that activate pylab, load gui input
     # handling and modify slightly %run

IPython/core/tests/test_application.py

@@ -4,7 +4,7 b''
 import os
 import tempfile
 
-from IPython.core.application import Application
+from IPython.core.application import BaseIPythonApplication
 from IPython.testing import decorators as testdec
 
 @testdec.onlyif_unicode_paths
@@ -16,22 +16,11 b' def test_unicode_cwd():'
     os.chdir(wd)
     #raise Exception(repr(os.getcwd()))
     try:
-        app = Application()
+        app = BaseIPythonApplication()
         # The lines below are copied from Application.initialize()
-        app.create_default_config()
-        app.log_default_config()
-        app.set_default_config_log_level()
-
-        # Find resources needed for filesystem access, using information from
-        # the above two
-        app.find_ipython_dir()
-        app.find_resources()
-        app.find_config_file_name()
-        app.find_config_file_paths()
-
-        # File-based config
-        app.pre_load_file_config()
-        app.load_file_config(suppress_errors=False)
+        app.init_profile_dir()
+        app.init_config_files()
+        app.load_config_file(suppress_errors=False)
     finally:
         os.chdir(old_wd)
 
@@ -48,22 +37,11 b' def test_unicode_ipdir():'
     old_ipdir2 = os.environ.pop("IPYTHON_DIR", None)
     os.environ["IPYTHONDIR"] = ipdir.encode("utf-8")
     try:
-        app = Application()
+        app = BaseIPythonApplication()
         # The lines below are copied from Application.initialize()
-        app.create_default_config()
-        app.log_default_config()
-        app.set_default_config_log_level()
-
-        # Find resources needed for filesystem access, using information from
-        # the above two
-        app.find_ipython_dir()
-        app.find_resources()
-        app.find_config_file_name()
-        app.find_config_file_paths()
-
-        # File-based config
-        app.pre_load_file_config()
-        app.load_file_config(suppress_errors=False)
+        app.init_profile_dir()
+        app.init_config_files()
+        app.load_config_file(suppress_errors=False)
     finally:
         if old_ipdir1:
             os.environ["IPYTHONDIR"] = old_ipdir1

IPython/extensions/parallelmagic.py

@@ -92,7 +92,7 b' class ParalleMagic(Plugin):'
         Then you can do the following::
 
             In [24]: %px a = 5
-            Parallel execution on engines: all
+            Parallel execution on engine(s): all
             Out[24]: 
             <Results List>
             [0] In [7]: a = 5
@@ -102,7 +102,7 b' class ParalleMagic(Plugin):'
         if self.active_view is None:
             print NO_ACTIVE_VIEW
             return
-        print "Parallel execution on engines: %s" % self.active_view.targets
+        print "Parallel execution on engine(s): %s" % self.active_view.targets
         result = self.active_view.execute(parameter_s, block=False)
         if self.active_view.block:
             result.get()
@@ -125,9 +125,9 b' class ParalleMagic(Plugin):'
             %autopx to enabled
 
             In [26]: a = 10
-            Parallel execution on engines: [0,1,2,3]
+            Parallel execution on engine(s): [0,1,2,3]
             In [27]: print a
-            Parallel execution on engines: [0,1,2,3]
+            Parallel execution on engine(s): [0,1,2,3]
             [stdout:0] 10
             [stdout:1] 10
             [stdout:2] 10
@@ -174,15 +174,21 b' class ParalleMagic(Plugin):'
         If self.active_view.block is True, wait for the result
         and display the result.  Otherwise, this is a noop.
         """
+        if isinstance(result.stdout, basestring):
+            # single result
+            stdouts = [result.stdout.rstrip()]
+        else:
+            stdouts = [s.rstrip() for s in result.stdout]
+        
         targets = self.active_view.targets
         if isinstance(targets, int):
             targets = [targets]
-        if targets == 'all':
+        elif targets == 'all':
             targets = self.active_view.client.ids
-        stdout = [s.rstrip() for s in result.stdout]
-        if any(stdout):
-            for i,eid in enumerate(targets):
-                print '[stdout:%i]'%eid, stdout[i]
+
+        if any(stdouts):
+            for eid,stdout in zip(targets, stdouts):
+                print '[stdout:%i]'%eid, stdout
 
 
     def pxrun_cell(self, raw_cell, store_history=True):

IPython/frontend/qt/base_frontend_mixin.py

@@ -30,8 +30,8 b' class BaseFrontendMixin(object):'
 
             # Disconnect the old kernel manager's channels.
             old_manager.sub_channel.message_received.disconnect(self._dispatch)
-            old_manager.xreq_channel.message_received.disconnect(self._dispatch)
-            old_manager.rep_channel.message_received.disconnect(self._dispatch)
+            old_manager.shell_channel.message_received.disconnect(self._dispatch)
+            old_manager.stdin_channel.message_received.disconnect(self._dispatch)
             old_manager.hb_channel.kernel_died.disconnect(
                 self._handle_kernel_died)
     
@@ -50,8 +50,8 b' class BaseFrontendMixin(object):'
 
         # Connect the new kernel manager's channels.
         kernel_manager.sub_channel.message_received.connect(self._dispatch)
-        kernel_manager.xreq_channel.message_received.connect(self._dispatch)
-        kernel_manager.rep_channel.message_received.connect(self._dispatch)
+        kernel_manager.shell_channel.message_received.connect(self._dispatch)
+        kernel_manager.stdin_channel.message_received.connect(self._dispatch)
         kernel_manager.hb_channel.kernel_died.connect(self._handle_kernel_died)
 
         # Handle the case where the kernel manager started channels before

IPython/frontend/qt/console/console_widget.py

@@ -19,7 +19,7 b' from IPython.external.qt import QtCore, QtGui'
 from IPython.config.configurable import Configurable
 from IPython.frontend.qt.rich_text import HtmlExporter
 from IPython.frontend.qt.util import MetaQObjectHasTraits, get_font
-from IPython.utils.traitlets import Bool, Enum, Int
+from IPython.utils.traitlets import Bool, Enum, Int, Unicode
 from ansi_code_processor import QtAnsiCodeProcessor
 from completion_widget import CompletionWidget
 from kill_ring import QtKillRing
@@ -55,33 +55,62 b' class ConsoleWidget(Configurable, QtGui.QWidget):'
 
     #------ Configuration ------------------------------------------------------
 
-    # Whether to process ANSI escape codes.
-    ansi_codes = Bool(True, config=True)
-
-    # The maximum number of lines of text before truncation. Specifying a
-    # non-positive number disables text truncation (not recommended).
-    buffer_size = Int(500, config=True)
-
-    # Whether to use a list widget or plain text output for tab completion.
-    gui_completion = Bool(False, config=True)
-
-    # The type of underlying text widget to use. Valid values are 'plain', which
-    # specifies a QPlainTextEdit, and 'rich', which specifies a QTextEdit.
+    ansi_codes = Bool(True, config=True,
+        help="Whether to process ANSI escape codes."
+    )
+    buffer_size = Int(500, config=True,
+        help="""
+        The maximum number of lines of text before truncation. Specifying a
+        non-positive number disables text truncation (not recommended).
+        """
+    )
+    gui_completion = Bool(False, config=True,
+        help="Use a list widget instead of plain text output for tab completion."
+    )
     # NOTE: this value can only be specified during initialization.
-    kind = Enum(['plain', 'rich'], default_value='plain', config=True)
-
-    # The type of paging to use. Valid values are:
-    #     'inside' : The widget pages like a traditional terminal.
-    #     'hsplit' : When paging is requested, the widget is split
-    #                horizontally. The top pane contains the console, and the
-    #                bottom pane contains the paged text.
-    #     'vsplit' : Similar to 'hsplit', except that a vertical splitter used.
-    #     'custom' : No action is taken by the widget beyond emitting a
-    #                'custom_page_requested(str)' signal.
-    #     'none'   : The text is written directly to the console.
+    kind = Enum(['plain', 'rich'], default_value='plain', config=True,
+        help="""
+        The type of underlying text widget to use. Valid values are 'plain', which
+        specifies a QPlainTextEdit, and 'rich', which specifies a QTextEdit.
+        """
+    )
     # NOTE: this value can only be specified during initialization.
     paging = Enum(['inside', 'hsplit', 'vsplit', 'custom', 'none'], 
-                  default_value='inside', config=True)
+                  default_value='inside', config=True,
+        help="""
+        The type of paging to use. Valid values are:
+        
+            'inside' : The widget pages like a traditional terminal.
+            'hsplit' : When paging is requested, the widget is split
+                       horizontally. The top pane contains the console, and the
+                       bottom pane contains the paged text.
+            'vsplit' : Similar to 'hsplit', except that a vertical splitter used.
+            'custom' : No action is taken by the widget beyond emitting a
+                       'custom_page_requested(str)' signal.
+            'none'   : The text is written directly to the console.
+        """)
+
+    font_family = Unicode(config=True,
+        help="""The font family to use for the console.
+        On OSX this defaults to Monaco, on Windows the default is
+        Consolas with fallback of Courier, and on other platforms
+        the default is Monospace.
+        """)
+    def _font_family_default(self):
+        if sys.platform == 'win32':
+            # Consolas ships with Vista/Win7, fallback to Courier if needed
+            return 'Consolas'
+        elif sys.platform == 'darwin':
+            # OSX always has Monaco, no need for a fallback
+            return 'Monaco'
+        else:
+            # Monospace should always exist, no need for a fallback
+            return 'Monospace'
+
+    font_size = Int(config=True,
+        help="""The font size. If unconfigured, Qt will be entrusted
+        with the size of the font.
+        """)
 
     # Whether to override ShortcutEvents for the keybindings defined by this
     # widget (Ctrl+n, Ctrl+a, etc). Enable this if you want this widget to take
@@ -591,16 +620,18 b' class ConsoleWidget(Configurable, QtGui.QWidget):'
         """
         if sys.platform == 'win32':
             # Consolas ships with Vista/Win7, fallback to Courier if needed
-            family, fallback = 'Consolas', 'Courier'
+            fallback = 'Courier'
         elif sys.platform == 'darwin':
-            # OSX always has Monaco, no need for a fallback
-            family, fallback = 'Monaco', None
+            # OSX always has Monaco
+            fallback = 'Monaco'
+        else:
+            # Monospace should always exist
+            fallback = 'Monospace'
+        font = get_font(self.font_family, fallback)
+        if self.font_size:
+            font.setPointSize(self.font_size)
         else:
-            # FIXME: remove Consolas as a default on Linux once our font
-            # selections are configurable by the user.
-            family, fallback = 'Consolas', 'Monospace'
-        font = get_font(family, fallback)
-        font.setPointSize(QtGui.qApp.font().pointSize())
+            font.setPointSize(QtGui.qApp.font().pointSize())
         font.setStyleHint(QtGui.QFont.TypeWriter)
         self._set_font(font)
 

IPython/frontend/qt/console/frontend_widget.py

@@ -13,7 +13,7 b' from IPython.external.qt import QtCore, QtGui'
 from IPython.core.inputsplitter import InputSplitter, transform_classic_prompt
 from IPython.core.oinspect import call_tip
 from IPython.frontend.qt.base_frontend_mixin import BaseFrontendMixin
-from IPython.utils.traitlets import Bool
+from IPython.utils.traitlets import Bool, Instance
 from bracket_matcher import BracketMatcher
 from call_tip_widget import CallTipWidget
 from completion_lexer import CompletionLexer
@@ -106,6 +106,7 b' class FrontendWidget(HistoryConsoleWidget, BaseFrontendMixin):'
     _ExecutionRequest = namedtuple('_ExecutionRequest', ['id', 'kind'])
     _input_splitter_class = InputSplitter
     _local_kernel = False
+    _highlighter = Instance(FrontendHighlighter)
 
     #---------------------------------------------------------------------------
     # 'object' interface
@@ -183,7 +184,7 b' class FrontendWidget(HistoryConsoleWidget, BaseFrontendMixin):'
 
         See parent class :meth:`execute` docstring for full details.
         """
-        msg_id = self.kernel_manager.xreq_channel.execute(source, hidden)
+        msg_id = self.kernel_manager.shell_channel.execute(source, hidden)
         self._request_info['execute'] = self._ExecutionRequest(msg_id, 'user')
         self._hidden = hidden
         if not hidden:
@@ -329,7 +330,7 b' class FrontendWidget(HistoryConsoleWidget, BaseFrontendMixin):'
         self.kernel_manager.sub_channel.flush()
 
         def callback(line):
-            self.kernel_manager.rep_channel.input(line)
+            self.kernel_manager.stdin_channel.input(line)
         self._readline(msg['content']['prompt'], callback=callback)
 
     def _handle_kernel_died(self, since_last_heartbeat):
@@ -526,7 +527,7 b' class FrontendWidget(HistoryConsoleWidget, BaseFrontendMixin):'
 
         # Send the metadata request to the kernel
         name = '.'.join(context)
-        msg_id = self.kernel_manager.xreq_channel.object_info(name)
+        msg_id = self.kernel_manager.shell_channel.object_info(name)
         pos = self._get_cursor().position()
         self._request_info['call_tip'] = self._CallTipRequest(msg_id, pos)
         return True
@@ -537,7 +538,7 b' class FrontendWidget(HistoryConsoleWidget, BaseFrontendMixin):'
         context = self._get_context()
         if context:
             # Send the completion request to the kernel
-            msg_id = self.kernel_manager.xreq_channel.complete(
+            msg_id = self.kernel_manager.shell_channel.complete(
                 '.'.join(context),                       # text
                 self._get_input_buffer_cursor_line(),    # line
                 self._get_input_buffer_cursor_column(),  # cursor_pos

IPython/frontend/qt/console/ipython_widget.py

@@ -23,9 +23,7 b' from IPython.core.inputsplitter import IPythonInputSplitter, \\'
 from IPython.core.usage import default_gui_banner
 from IPython.utils.traitlets import Bool, Str, Unicode
 from frontend_widget import FrontendWidget
-from styles import (default_light_style_sheet,  default_light_syntax_style,
-                    default_dark_style_sheet, default_dark_syntax_style,
-                    default_bw_style_sheet, default_bw_syntax_style)
+import styles
 
 #-----------------------------------------------------------------------------
 # Constants
@@ -42,6 +40,11 b" default_output_sep2 = ''"
 # Base path for most payload sources.
 zmq_shell_source = 'IPython.zmq.zmqshell.ZMQInteractiveShell'
 
+if sys.platform.startswith('win'):
+    default_editor = 'notepad'
+else:
+    default_editor = ''
+
 #-----------------------------------------------------------------------------
 # IPythonWidget class
 #-----------------------------------------------------------------------------
@@ -56,26 +59,35 b' class IPythonWidget(FrontendWidget):'
     custom_edit = Bool(False)
     custom_edit_requested = QtCore.Signal(object, object)
 
-    # A command for invoking a system text editor. If the string contains a
-    # {filename} format specifier, it will be used. Otherwise, the filename will
-    # be appended to the end the command.
-    editor = Unicode('default', config=True)
-
-    # The editor command to use when a specific line number is requested. The
-    # string should contain two format specifiers: {line} and {filename}. If
-    # this parameter is not specified, the line number option to the %edit magic
-    # will be ignored.
-    editor_line = Unicode(config=True)
-
-    # A CSS stylesheet. The stylesheet can contain classes for:
-    #     1. Qt: QPlainTextEdit, QFrame, QWidget, etc
-    #     2. Pygments: .c, .k, .o, etc (see PygmentsHighlighter)
-    #     3. IPython: .error, .in-prompt, .out-prompt, etc
-    style_sheet = Unicode(config=True)
+    editor = Unicode(default_editor, config=True,
+        help="""
+        A command for invoking a system text editor. If the string contains a
+        {filename} format specifier, it will be used. Otherwise, the filename will
+        be appended to the end the command.
+        """)
+
+    editor_line = Unicode(config=True,
+        help="""
+        The editor command to use when a specific line number is requested. The
+        string should contain two format specifiers: {line} and {filename}. If
+        this parameter is not specified, the line number option to the %edit magic
+        will be ignored.
+        """)
+
+    style_sheet = Unicode(config=True,
+        help="""
+        A CSS stylesheet. The stylesheet can contain classes for:
+            1. Qt: QPlainTextEdit, QFrame, QWidget, etc
+            2. Pygments: .c, .k, .o, etc. (see PygmentsHighlighter)
+            3. IPython: .error, .in-prompt, .out-prompt, etc
+        """)
     
-    # If not empty, use this Pygments style for syntax highlighting. Otherwise,
-    # the style sheet is queried for Pygments style information.
-    syntax_style = Str(config=True)
+
+    syntax_style = Str(config=True,
+        help="""
+        If not empty, use this Pygments style for syntax highlighting. Otherwise,
+        the style sheet is queried for Pygments style information.
+        """)
 
     # Prompts.
     in_prompt = Str(default_in_prompt, config=True)
@@ -215,7 +227,7 b' class IPythonWidget(FrontendWidget):'
         """ Reimplemented to make a history request.
         """
         super(IPythonWidget, self)._started_channels()
-        self.kernel_manager.xreq_channel.history(hist_access_type='tail', n=1000)
+        self.kernel_manager.shell_channel.history(hist_access_type='tail', n=1000)
 
     #---------------------------------------------------------------------------
     # 'ConsoleWidget' public interface
@@ -257,7 +269,7 b' class IPythonWidget(FrontendWidget):'
         text = ''
         
         # Send the completion request to the kernel
-        msg_id = self.kernel_manager.xreq_channel.complete(
+        msg_id = self.kernel_manager.shell_channel.complete(
             text,                                    # text
             self._get_input_buffer_cursor_line(),    # line
             self._get_input_buffer_cursor_column(),  # cursor_pos
@@ -308,7 +320,7 b' class IPythonWidget(FrontendWidget):'
         """
         # If a number was not specified, make a prompt number request.
         if number is None:
-            msg_id = self.kernel_manager.xreq_channel.execute('', silent=True)
+            msg_id = self.kernel_manager.shell_channel.execute('', silent=True)
             info = self._ExecutionRequest(msg_id, 'prompt')
             self._request_info['execute'] = info
             return
@@ -371,14 +383,14 b' class IPythonWidget(FrontendWidget):'
         """
         colors = colors.lower()
         if colors=='lightbg':
-            self.style_sheet = default_light_style_sheet
-            self.syntax_style = default_light_syntax_style
+            self.style_sheet = styles.default_light_style_sheet
+            self.syntax_style = styles.default_light_syntax_style
         elif colors=='linux':
-            self.style_sheet = default_dark_style_sheet
-            self.syntax_style = default_dark_syntax_style
+            self.style_sheet = styles.default_dark_style_sheet
+            self.syntax_style = styles.default_dark_syntax_style
         elif colors=='nocolor':
-            self.style_sheet = default_bw_style_sheet
-            self.syntax_style = default_bw_syntax_style
+            self.style_sheet = styles.default_bw_style_sheet
+            self.syntax_style = styles.default_bw_syntax_style
         else:
             raise KeyError("No such color scheme: %s"%colors)
 
@@ -399,8 +411,10 b' class IPythonWidget(FrontendWidget):'
         """
         if self.custom_edit:
             self.custom_edit_requested.emit(filename, line)
-        elif self.editor == 'default':
-            self._append_plain_text('No default editor available.\n')
+        elif not self.editor:
+            self._append_plain_text('No default editor available.\n'
+            'Specify a GUI text editor in the `IPythonWidget.editor` configurable\n'
+            'to enable the %edit magic')
         else:
             try:
                 filename = '"%s"' % filename
@@ -482,9 +496,13 b' class IPythonWidget(FrontendWidget):'
         bg_color = self._control.palette().window().color()
         self._ansi_processor.set_background_color(bg_color)
 
+
     def _syntax_style_changed(self):
         """ Set the style for the syntax highlighter.
         """
+        if self._highlighter is None:
+            # ignore premature calls
+            return
         if self.syntax_style:
             self._highlighter.set_style(self.syntax_style)
         else:

IPython/frontend/qt/console/qtconsoleapp.py

@@ -1,21 +1,50 b''
 """ A minimal application using the Qt console-style IPython frontend.
+
+This is not a complete console app, as subprocess will not be able to receive
+input, there is no real readline support, among other limitations.
+
+Authors:
+
+* Evan Patterson
+* Min RK
+* Erik Tollerud
+* Fernando Perez
+
 """
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
-# Systemm library imports
+# stdlib imports
+import os
+import signal
+import sys
+
+# System library imports
 from IPython.external.qt import QtGui
 from pygments.styles import get_all_styles
 
 # Local imports
-from IPython.external.argparse import ArgumentParser
+from IPython.config.application import boolean_flag
+from IPython.core.application import BaseIPythonApplication
+from IPython.core.profiledir import ProfileDir
 from IPython.frontend.qt.console.frontend_widget import FrontendWidget
 from IPython.frontend.qt.console.ipython_widget import IPythonWidget
 from IPython.frontend.qt.console.rich_ipython_widget import RichIPythonWidget
 from IPython.frontend.qt.console import styles
 from IPython.frontend.qt.kernelmanager import QtKernelManager
+from IPython.utils.traitlets import (
+    Dict, List, Unicode, Int, CaselessStrEnum, CBool, Any
+)
+from IPython.zmq.ipkernel import (
+    flags as ipkernel_flags,
+    aliases as ipkernel_aliases,
+    IPKernelApp
+)
+from IPython.zmq.session import Session
+from IPython.zmq.zmqshell import ZMQInteractiveShell
+
 
 #-----------------------------------------------------------------------------
 # Network Constants
@@ -33,7 +62,8 b' class MainWindow(QtGui.QMainWindow):'
     # 'object' interface
     #---------------------------------------------------------------------------
     
-    def __init__(self, app, frontend, existing=False, may_close=True):
+    def __init__(self, app, frontend, existing=False, may_close=True,
+                    confirm_exit=True):
         """ Create a MainWindow for the specified FrontendWidget.
         
         The app is passed as an argument to allow for different
@@ -52,6 +82,7 b' class MainWindow(QtGui.QMainWindow):'
         else:
             self._may_close = True
         self._frontend.exit_requested.connect(self.close)
+        self._confirm_exit = confirm_exit
         self.setCentralWidget(frontend)
     
     #---------------------------------------------------------------------------
@@ -71,6 +102,11 b' class MainWindow(QtGui.QMainWindow):'
         
         kernel_manager = self._frontend.kernel_manager
         
+        if keepkernel is None and not self._confirm_exit:
+            # don't prompt, just terminate the kernel if we own it
+            # or leave it alone if we don't
+            keepkernel = not self._existing
+        
         if keepkernel is None: #show prompt
             if kernel_manager and kernel_manager.channels_running:
                 title = self.window().windowTitle()
@@ -127,123 +163,216 b' class MainWindow(QtGui.QMainWindow):'
                 event.accept()
 
 #-----------------------------------------------------------------------------
-# Main entry point
+# Aliases and Flags
 #-----------------------------------------------------------------------------
 
-def main():
-    """ Entry point for application.
+flags = dict(ipkernel_flags)
+
+flags.update({
+    'existing' : ({'IPythonQtConsoleApp' : {'existing' : True}},
+            "Connect to an existing kernel."),
+    'pure' : ({'IPythonQtConsoleApp' : {'pure' : True}},
+            "Use a pure Python kernel instead of an IPython kernel."),
+    'plain' : ({'ConsoleWidget' : {'kind' : 'plain'}},
+            "Disable rich text support."),
+})
+flags.update(boolean_flag(
+    'gui-completion', 'ConsoleWidget.gui_completion',
+    "use a GUI widget for tab completion",
+    "use plaintext output for completion"
+))
+flags.update(boolean_flag(
+    'confirm-exit', 'IPythonQtConsoleApp.confirm_exit',
+    """Set to display confirmation dialog on exit. You can always use 'exit' or 'quit',
+       to force a direct exit without any confirmation.
+    """,
+    """Don't prompt the user when exiting. This will terminate the kernel
+       if it is owned by the frontend, and leave it alive if it is external.
     """
-    # Parse command line arguments.
-    parser = ArgumentParser()
-    kgroup = parser.add_argument_group('kernel options')
-    kgroup.add_argument('-e', '--existing', action='store_true',
-                        help='connect to an existing kernel')
-    kgroup.add_argument('--ip', type=str, default=LOCALHOST,
-                        help=\
-            "set the kernel\'s IP address [default localhost].\
-            If the IP address is something other than localhost, then \
-            Consoles on other machines will be able to connect\
-            to the Kernel, so be careful!")
-    kgroup.add_argument('--xreq', type=int, metavar='PORT', default=0,
-                        help='set the XREQ channel port [default random]')
-    kgroup.add_argument('--sub', type=int, metavar='PORT', default=0,
-                        help='set the SUB channel port [default random]')
-    kgroup.add_argument('--rep', type=int, metavar='PORT', default=0,
-                        help='set the REP channel port [default random]')
-    kgroup.add_argument('--hb', type=int, metavar='PORT', default=0,
-                        help='set the heartbeat port [default random]')
-
-    egroup = kgroup.add_mutually_exclusive_group()
-    egroup.add_argument('--pure', action='store_true', help = \
-                        'use a pure Python kernel instead of an IPython kernel')
-    egroup.add_argument('--pylab', type=str, metavar='GUI', nargs='?', 
-                       const='auto', help = \
-        "Pre-load matplotlib and numpy for interactive use. If GUI is not \
-         given, the GUI backend is matplotlib's, otherwise use one of: \
-         ['tk', 'gtk', 'qt', 'wx', 'inline'].")
-
-    wgroup = parser.add_argument_group('widget options')
-    wgroup.add_argument('--paging', type=str, default='inside',
-                        choices = ['inside', 'hsplit', 'vsplit', 'none'],
-                        help='set the paging style [default inside]')
-    wgroup.add_argument('--plain', action='store_true',
-                        help='disable rich text support')
-    wgroup.add_argument('--gui-completion', action='store_true',
-                        help='use a GUI widget for tab completion')
-    wgroup.add_argument('--style', type=str,
-                        choices = list(get_all_styles()),
-                        help='specify a pygments style for by name')
-    wgroup.add_argument('--stylesheet', type=str,
-                        help='path to a custom CSS stylesheet')
-    wgroup.add_argument('--colors', type=str, help = \
-        "Set the color scheme (LightBG,Linux,NoColor). This is guessed \
-         based on the pygments style if not set.")
-
-    args = parser.parse_args()
-
-    # parse the colors arg down to current known labels
-    if args.colors:
-        colors=args.colors.lower()
-        if colors in ('lightbg', 'light'):
-            colors='lightbg'
-        elif colors in ('dark', 'linux'):
-            colors='linux'
-        else:
-            colors='nocolor'
-    elif args.style:
-        if args.style=='bw':
-            colors='nocolor'
-        elif styles.dark_style(args.style):
-            colors='linux'
+))
+# the flags that are specific to the frontend
+# these must be scrubbed before being passed to the kernel,
+# or it will raise an error on unrecognized flags
+qt_flags = ['existing', 'pure', 'plain', 'gui-completion', 'no-gui-completion',
+            'confirm-exit', 'no-confirm-exit']
+
+aliases = dict(ipkernel_aliases)
+
+aliases.update(dict(
+    hb = 'IPythonQtConsoleApp.hb_port',
+    shell = 'IPythonQtConsoleApp.shell_port',
+    iopub = 'IPythonQtConsoleApp.iopub_port',
+    stdin = 'IPythonQtConsoleApp.stdin_port',
+    ip = 'IPythonQtConsoleApp.ip',
+
+    plain = 'IPythonQtConsoleApp.plain',
+    pure = 'IPythonQtConsoleApp.pure',
+    gui_completion = 'ConsoleWidget.gui_completion',
+    style = 'IPythonWidget.syntax_style',
+    stylesheet = 'IPythonQtConsoleApp.stylesheet',
+    colors = 'ZMQInteractiveShell.colors',
+
+    editor = 'IPythonWidget.editor',
+))
+
+#-----------------------------------------------------------------------------
+# IPythonQtConsole
+#-----------------------------------------------------------------------------
+class IPythonQtConsoleApp(BaseIPythonApplication):
+    name = 'ipython-qtconsole'
+    default_config_file_name='ipython_config.py'
+    
+    description = """
+        The IPython QtConsole.
+        
+        This launches a Console-style application using Qt.  It is not a full
+        console, in that launched terminal subprocesses will not.
+        
+        The QtConsole supports various extra features beyond the
+        
+    """
+    
+    classes = [IPKernelApp, IPythonWidget, ZMQInteractiveShell, ProfileDir, Session]
+    flags = Dict(flags)
+    aliases = Dict(aliases)
+
+    kernel_argv = List(Unicode)
+
+    # connection info:
+    ip = Unicode(LOCALHOST, config=True,
+        help="""Set the kernel\'s IP address [default localhost].
+        If the IP address is something other than localhost, then
+        Consoles on other machines will be able to connect
+        to the Kernel, so be careful!"""
+    )
+    hb_port = Int(0, config=True,
+        help="set the heartbeat port [default: random]")
+    shell_port = Int(0, config=True,
+        help="set the shell (XREP) port [default: random]")
+    iopub_port = Int(0, config=True,
+        help="set the iopub (PUB) port [default: random]")
+    stdin_port = Int(0, config=True,
+        help="set the stdin (XREQ) port [default: random]")
+
+    existing = CBool(False, config=True,
+        help="Whether to connect to an already running Kernel.")
+
+    stylesheet = Unicode('', config=True,
+        help="path to a custom CSS stylesheet")
+
+    pure = CBool(False, config=True,
+        help="Use a pure Python kernel instead of an IPython kernel.")
+    plain = CBool(False, config=True,
+        help="Use a plaintext widget instead of rich text (plain can't print/save).")
+
+    def _pure_changed(self, name, old, new):
+        kind = 'plain' if self.plain else 'rich'
+        self.config.ConsoleWidget.kind = kind
+        if self.pure:
+            self.widget_factory = FrontendWidget
+        elif self.plain:
+            self.widget_factory = IPythonWidget
         else:
-            colors='lightbg'
-    else:
-        colors=None
-
-    # Don't let Qt or ZMQ swallow KeyboardInterupts.
-    import signal
-    signal.signal(signal.SIGINT, signal.SIG_DFL)
-
-    # Create a KernelManager and start a kernel.
-    kernel_manager = QtKernelManager(xreq_address=(args.ip, args.xreq),
-                                     sub_address=(args.ip, args.sub),
-                                     rep_address=(args.ip, args.rep),
-                                     hb_address=(args.ip, args.hb))
-    if not args.existing:
-        # if not args.ip in LOCAL_IPS+ALL_ALIAS:
-        #     raise ValueError("Must bind a local ip, such as: %s"%LOCAL_IPS)
-
-        kwargs = dict(ip=args.ip)
-        if args.pure:
-            kwargs['ipython']=False
+            self.widget_factory = RichIPythonWidget
+
+    _plain_changed = _pure_changed
+
+    confirm_exit = CBool(True, config=True,
+        help="""
+        Set to display confirmation dialog on exit. You can always use 'exit' or 'quit',
+        to force a direct exit without any confirmation.""",
+    )
+    
+    # the factory for creating a widget
+    widget_factory = Any(RichIPythonWidget)
+
+    def parse_command_line(self, argv=None):
+        super(IPythonQtConsoleApp, self).parse_command_line(argv)
+        if argv is None:
+            argv = sys.argv[1:]
+
+        self.kernel_argv = list(argv) # copy
+
+        # scrub frontend-specific flags
+        for a in argv:
+            if a.startswith('--') and a[2:] in qt_flags:
+                self.kernel_argv.remove(a)
+
+    def init_kernel_manager(self):
+        # Don't let Qt or ZMQ swallow KeyboardInterupts.
+        signal.signal(signal.SIGINT, signal.SIG_DFL)
+
+        # Create a KernelManager and start a kernel.
+        self.kernel_manager = QtKernelManager(
+                                shell_address=(self.ip, self.shell_port),
+                                sub_address=(self.ip, self.iopub_port),
+                                stdin_address=(self.ip, self.stdin_port),
+                                hb_address=(self.ip, self.hb_port),
+                                config=self.config
+        )
+        # start the kernel
+        if not self.existing:
+            kwargs = dict(ip=self.ip, ipython=not self.pure)
+            kwargs['extra_arguments'] = self.kernel_argv
+            self.kernel_manager.start_kernel(**kwargs)
+        self.kernel_manager.start_channels()
+
+
+    def init_qt_elements(self):
+        # Create the widget.
+        self.app = QtGui.QApplication([])
+        local_kernel = (not self.existing) or self.ip in LOCAL_IPS
+        self.widget = self.widget_factory(config=self.config,
+                                        local_kernel=local_kernel)
+        self.widget.kernel_manager = self.kernel_manager
+        self.window = MainWindow(self.app, self.widget, self.existing,
+                                may_close=local_kernel,
+                                confirm_exit=self.confirm_exit)
+        self.window.setWindowTitle('Python' if self.pure else 'IPython')
+
+    def init_colors(self):
+        """Configure the coloring of the widget"""
+        # Note: This will be dramatically simplified when colors
+        # are removed from the backend.
+
+        if self.pure:
+            # only IPythonWidget supports styling
+            return
+
+        # parse the colors arg down to current known labels
+        try:
+            colors = self.config.ZMQInteractiveShell.colors
+        except AttributeError:
+            colors = None
+        try:
+            style = self.config.IPythonWidget.colors
+        except AttributeError:
+            style = None
+
+        # find the value for colors:
+        if colors:
+            colors=colors.lower()
+            if colors in ('lightbg', 'light'):
+                colors='lightbg'
+            elif colors in ('dark', 'linux'):
+                colors='linux'
+            else:
+                colors='nocolor'
+        elif style:
+            if style=='bw':
+                colors='nocolor'
+            elif styles.dark_style(style):
+                colors='linux'
+            else:
+                colors='lightbg'
         else:
-            kwargs['colors']=colors
-            if args.pylab:
-                kwargs['pylab']=args.pylab
-
-        kernel_manager.start_kernel(**kwargs)
-    kernel_manager.start_channels()
-
-    # Create the widget.
-    app = QtGui.QApplication([])
-    local_kernel = (not args.existing) or args.ip in LOCAL_IPS
-    if args.pure:
-        kind = 'plain' if args.plain else 'rich'
-        widget = FrontendWidget(kind=kind, paging=args.paging, 
-                                local_kernel=local_kernel)
-    elif args.plain:
-        widget = IPythonWidget(paging=args.paging, local_kernel=local_kernel)
-    else:
-        widget = RichIPythonWidget(paging=args.paging, 
-                                   local_kernel=local_kernel)
-    widget.gui_completion = args.gui_completion
-    widget.kernel_manager = kernel_manager
-
-    # Configure the style.
-    if not args.pure: # only IPythonWidget supports styles
-        if args.style:
-            widget.syntax_style = args.style
-            widget.style_sheet = styles.sheet_from_template(args.style, colors)
+            colors=None
+
+        # Configure the style.
+        widget = self.widget
+        if style:
+            widget.style_sheet = styles.sheet_from_template(style, colors)
+            widget.syntax_style = style
             widget._syntax_style_changed()
             widget._style_sheet_changed()
         elif colors:
@@ -254,23 +383,38 b' def main():'
             # defaults to change
             widget.set_default_style()
 
-        if args.stylesheet:
+        if self.stylesheet:
             # we got an expicit stylesheet
-            if os.path.isfile(args.stylesheet):
-                with open(args.stylesheet) as f:
+            if os.path.isfile(self.stylesheet):
+                with open(self.stylesheet) as f:
                     sheet = f.read()
                 widget.style_sheet = sheet
                 widget._style_sheet_changed()
             else:
-                raise IOError("Stylesheet %r not found."%args.stylesheet)
+                raise IOError("Stylesheet %r not found."%self.stylesheet)
+
+    def initialize(self, argv=None):
+        super(IPythonQtConsoleApp, self).initialize(argv)
+        self.init_kernel_manager()
+        self.init_qt_elements()
+        self.init_colors()
+
+    def start(self):
 
-    # Create the main window.
-    window = MainWindow(app, widget, args.existing, may_close=local_kernel)
-    window.setWindowTitle('Python' if args.pure else 'IPython')
-    window.show()
+        # draw the window
+        self.window.show()
 
-    # Start the application main loop.
-    app.exec_()
+        # Start the application main loop.
+        self.app.exec_()
+
+#-----------------------------------------------------------------------------
+# Main entry point
+#-----------------------------------------------------------------------------
+
+def main():
+    app = IPythonQtConsoleApp()
+    app.initialize()
+    app.start()
 
 
 if __name__ == '__main__':

IPython/frontend/qt/kernelmanager.py

@@ -7,7 +7,7 b' from IPython.external.qt import QtCore'
 # IPython imports.
 from IPython.utils.traitlets import Type
 from IPython.zmq.kernelmanager import KernelManager, SubSocketChannel, \
-    XReqSocketChannel, RepSocketChannel, HBSocketChannel
+    ShellSocketChannel, StdInSocketChannel, HBSocketChannel
 from util import MetaQObjectHasTraits, SuperQObject
 
 
@@ -20,7 +20,7 b' class SocketChannelQObject(SuperQObject):'
     stopped = QtCore.Signal()
 
     #---------------------------------------------------------------------------
-    # 'ZmqSocketChannel' interface
+    # 'ZMQSocketChannel' interface
     #---------------------------------------------------------------------------
 
     def start(self):
@@ -36,7 +36,7 b' class SocketChannelQObject(SuperQObject):'
         self.stopped.emit()
 
 
-class QtXReqSocketChannel(SocketChannelQObject, XReqSocketChannel):
+class QtShellSocketChannel(SocketChannelQObject, ShellSocketChannel):
 
     # Emitted when any message is received.
     message_received = QtCore.Signal(object)
@@ -56,7 +56,7 b' class QtXReqSocketChannel(SocketChannelQObject, XReqSocketChannel):'
     _handlers_called = False
 
     #---------------------------------------------------------------------------
-    # 'XReqSocketChannel' interface
+    # 'ShellSocketChannel' interface
     #---------------------------------------------------------------------------
     
     def call_handlers(self, msg):
@@ -76,7 +76,7 b' class QtXReqSocketChannel(SocketChannelQObject, XReqSocketChannel):'
             self._handlers_called = True
 
     #---------------------------------------------------------------------------
-    # 'QtXReqSocketChannel' interface
+    # 'QtShellSocketChannel' interface
     #---------------------------------------------------------------------------
 
     def reset_first_reply(self):
@@ -136,7 +136,7 b' class QtSubSocketChannel(SocketChannelQObject, SubSocketChannel):'
         QtCore.QCoreApplication.instance().processEvents()
 
 
-class QtRepSocketChannel(SocketChannelQObject, RepSocketChannel):
+class QtStdInSocketChannel(SocketChannelQObject, StdInSocketChannel):
 
     # Emitted when any message is received.
     message_received = QtCore.Signal(object)
@@ -145,7 +145,7 b' class QtRepSocketChannel(SocketChannelQObject, RepSocketChannel):'
     input_requested = QtCore.Signal(object)
 
     #---------------------------------------------------------------------------
-    # 'RepSocketChannel' interface
+    # 'StdInSocketChannel' interface
     #---------------------------------------------------------------------------
 
     def call_handlers(self, msg):
@@ -190,8 +190,8 b' class QtKernelManager(KernelManager, SuperQObject):'
 
     # Use Qt-specific channel classes that emit signals.
     sub_channel_class = Type(QtSubSocketChannel)
-    xreq_channel_class = Type(QtXReqSocketChannel)
-    rep_channel_class = Type(QtRepSocketChannel)
+    shell_channel_class = Type(QtShellSocketChannel)
+    stdin_channel_class = Type(QtStdInSocketChannel)
     hb_channel_class = Type(QtHBSocketChannel)
 
     #---------------------------------------------------------------------------
@@ -203,8 +203,8 b' class QtKernelManager(KernelManager, SuperQObject):'
     def start_kernel(self, *args, **kw):
         """ Reimplemented for proper heartbeat management.
         """
-        if self._xreq_channel is not None:
-            self._xreq_channel.reset_first_reply()
+        if self._shell_channel is not None:
+            self._shell_channel.reset_first_reply()
         super(QtKernelManager, self).start_kernel(*args, **kw)
 
     #------ Channel management -------------------------------------------------
@@ -222,13 +222,13 b' class QtKernelManager(KernelManager, SuperQObject):'
         self.stopped_channels.emit()
 
     @property
-    def xreq_channel(self):
+    def shell_channel(self):
         """ Reimplemented for proper heartbeat management.
         """
-        if self._xreq_channel is None:
-            self._xreq_channel = super(QtKernelManager, self).xreq_channel
-            self._xreq_channel.first_reply.connect(self._first_reply)
-        return self._xreq_channel
+        if self._shell_channel is None:
+            self._shell_channel = super(QtKernelManager, self).shell_channel
+            self._shell_channel.first_reply.connect(self._first_reply)
+        return self._shell_channel
 
     #---------------------------------------------------------------------------
     # Protected interface

IPython/frontend/terminal/interactiveshell.py

@@ -58,11 +58,21 b' raw_input_original = raw_input'
 
 class TerminalInteractiveShell(InteractiveShell):
 
-    autoedit_syntax = CBool(False, config=True)
+    autoedit_syntax = CBool(False, config=True,
+        help="auto editing of files with syntax errors.")
     banner = Unicode('')
-    banner1 = Unicode(default_banner, config=True)
-    banner2 = Unicode('', config=True)
-    confirm_exit = CBool(True, config=True)
+    banner1 = Unicode(default_banner, config=True,
+        help="""The part of the banner to be printed before the profile"""
+    )
+    banner2 = Unicode('', config=True,
+        help="""The part of the banner to be printed after the profile"""
+    )
+    confirm_exit = CBool(True, config=True,
+        help="""
+        Set to confirm when you try to exit IPython with an EOF (Control-D
+        in Unix, Control-Z/Enter in Windows). By typing 'exit' or 'quit',
+        you can force a direct exit without any confirmation.""",
+    )
     # This display_banner only controls whether or not self.show_banner()
     # is called when mainloop/interact are called.  The default is False
     # because for the terminal based application, the banner behavior
@@ -71,19 +81,35 b' class TerminalInteractiveShell(InteractiveShell):'
     display_banner = CBool(False) # This isn't configurable!
     embedded = CBool(False)
     embedded_active = CBool(False)
-    editor = Unicode(get_default_editor(), config=True)
-    pager = Unicode('less', config=True)
-
-    screen_length = Int(0, config=True)
-    term_title = CBool(False, config=True)
-
-    def __init__(self, config=None, ipython_dir=None, user_ns=None,
+    editor = Unicode(get_default_editor(), config=True,
+        help="Set the editor used by IPython (default to $EDITOR/vi/notepad)."
+    )
+    pager = Unicode('less', config=True,
+        help="The shell program to be used for paging.")
+
+    screen_length = Int(0, config=True,
+        help=
+        """Number of lines of your screen, used to control printing of very
+        long strings.  Strings longer than this number of lines will be sent
+        through a pager instead of directly printed.  The default value for
+        this is 0, which means IPython will auto-detect your screen size every
+        time it needs to print certain potentially long strings (this doesn't
+        change the behavior of the 'print' keyword, it's only triggered
+        internally). If for some reason this isn't working well (it needs
+        curses support), specify it yourself. Otherwise don't change the
+        default.""",
+    )
+    term_title = CBool(False, config=True,
+        help="Enable auto setting the terminal title."
+    )
+
+    def __init__(self, config=None, ipython_dir=None, profile_dir=None, user_ns=None,
                  user_global_ns=None, custom_exceptions=((),None),
                  usage=None, banner1=None, banner2=None,
                  display_banner=None):
 
         super(TerminalInteractiveShell, self).__init__(
-            config=config, ipython_dir=ipython_dir, user_ns=user_ns,
+            config=config, profile_dir=profile_dir, user_ns=user_ns,
             user_global_ns=user_global_ns, custom_exceptions=custom_exceptions
         )
         # use os.system instead of utils.process.system by default, except on Windows
@@ -167,7 +193,7 b' class TerminalInteractiveShell(InteractiveShell):'
 
     def compute_banner(self):
         self.banner = self.banner1
-        if self.profile:
+        if self.profile and self.profile != 'default':
             self.banner += '\nIPython profile: %s\n' % self.profile
         if self.banner2:
             self.banner += '\n' + self.banner2

IPython/frontend/terminal/ipapp.py

@@ -9,6 +9,7 b' Authors'
 
 * Brian Granger
 * Fernando Perez
+* Min Ragan-Kelley
 """
 
 #-----------------------------------------------------------------------------
@@ -28,17 +29,26 b' import logging'
 import os
 import sys
 
+from IPython.config.loader import (
+    Config, PyFileConfigLoader
+)
+from IPython.config.application import boolean_flag
 from IPython.core import release
+from IPython.core import usage
 from IPython.core.crashhandler import CrashHandler
-from IPython.core.application import Application, BaseAppConfigLoader
-from IPython.frontend.terminal.interactiveshell import TerminalInteractiveShell
-from IPython.config.loader import (
-    Config,
-    PyFileConfigLoader
+from IPython.core.formatters import PlainTextFormatter
+from IPython.core.application import (
+    ProfileDir, BaseIPythonApplication, base_flags, base_aliases
+)
+from IPython.core.shellapp import (
+    InteractiveShellApp, shell_flags, shell_aliases
 )
+from IPython.frontend.terminal.interactiveshell import TerminalInteractiveShell
 from IPython.lib import inputhook
-from IPython.utils.path import filefind, get_ipython_dir, check_for_old_config
-from IPython.core import usage
+from IPython.utils.path import get_ipython_dir, check_for_old_config
+from IPython.utils.traitlets import (
+    Bool, Dict, CaselessStrEnum
+)
 
 #-----------------------------------------------------------------------------
 # Globals, utilities and helpers
@@ -48,276 +58,6 b' from IPython.core import usage'
 default_config_file_name = u'ipython_config.py'
 
 
-class IPAppConfigLoader(BaseAppConfigLoader):
-
-    def _add_arguments(self):
-        super(IPAppConfigLoader, self)._add_arguments()
-        paa = self.parser.add_argument
-        paa('-p', 
-            '--profile', dest='Global.profile', type=unicode,
-            help=
-            """The string name of the ipython profile to be used.  Assume that your
-            config file is ipython_config-<name>.py (looks in current dir first,
-            then in IPYTHON_DIR). This is a quick way to keep and load multiple
-            config files for different tasks, especially if include your basic one
-            in your more specialized ones.  You can keep a basic
-            IPYTHON_DIR/ipython_config.py file and then have other 'profiles' which
-            include this one and load extra things for particular tasks.""",
-            metavar='Global.profile')
-        paa('--config-file', 
-            dest='Global.config_file', type=unicode,
-            help=
-            """Set the config file name to override default.  Normally IPython
-            loads ipython_config.py (from current directory) or
-            IPYTHON_DIR/ipython_config.py.  If the loading of your config file
-            fails, IPython starts with a bare bones configuration (no modules
-            loaded at all).""",
-            metavar='Global.config_file')
-        paa('--autocall', 
-            dest='InteractiveShell.autocall', type=int, 
-            help=
-            """Make IPython automatically call any callable object even if you
-            didn't type explicit parentheses. For example, 'str 43' becomes
-            'str(43)' automatically.  The value can be '0' to disable the feature,
-            '1' for 'smart' autocall, where it is not applied if there are no more
-            arguments on the line, and '2' for 'full' autocall, where all callable
-            objects are automatically called (even if no arguments are present).
-            The default is '1'.""",
-            metavar='InteractiveShell.autocall')
-        paa('--autoindent',
-            action='store_true', dest='InteractiveShell.autoindent',
-            help='Turn on autoindenting.')
-        paa('--no-autoindent',
-            action='store_false', dest='InteractiveShell.autoindent',
-            help='Turn off autoindenting.')
-        paa('--automagic',
-            action='store_true', dest='InteractiveShell.automagic',
-            help=
-            """Turn on the auto calling of magic commands. Type %%magic at the
-            IPython  prompt  for  more information.""")
-        paa('--no-automagic',
-            action='store_false', dest='InteractiveShell.automagic',
-            help='Turn off the auto calling of magic commands.')
-        paa('--autoedit-syntax',
-            action='store_true', dest='TerminalInteractiveShell.autoedit_syntax',
-            help='Turn on auto editing of files with syntax errors.')
-        paa('--no-autoedit-syntax',
-            action='store_false', dest='TerminalInteractiveShell.autoedit_syntax',
-            help='Turn off auto editing of files with syntax errors.')
-        paa('--banner',
-            action='store_true', dest='Global.display_banner',
-            help='Display a banner upon starting IPython.')
-        paa('--no-banner',
-            action='store_false', dest='Global.display_banner',
-            help="Don't display a banner upon starting IPython.")
-        paa('--cache-size',
-            type=int, dest='InteractiveShell.cache_size',
-            help=
-            """Set the size of the output cache.  The default is 1000, you can
-            change it permanently in your config file.  Setting it to 0 completely
-            disables the caching system, and the minimum value accepted is 20 (if
-            you provide a value less than 20, it is reset to 0 and a warning is
-            issued).  This limit is defined because otherwise you'll spend more
-            time re-flushing a too small cache than working""",
-            metavar='InteractiveShell.cache_size')
-        paa('--classic',
-            action='store_true', dest='Global.classic',
-            help="Gives IPython a similar feel to the classic Python prompt.")
-        paa('--colors',
-            type=str, dest='InteractiveShell.colors',
-            help="Set the color scheme (NoColor, Linux, and LightBG).",
-            metavar='InteractiveShell.colors')
-        paa('--color-info',
-            action='store_true', dest='InteractiveShell.color_info',
-            help=
-            """IPython can display information about objects via a set of func-
-            tions, and optionally can use colors for this, syntax highlighting
-            source code and various other elements.  However, because this
-            information is passed through a pager (like 'less') and many pagers get
-            confused with color codes, this option is off by default.  You can test
-            it and turn it on permanently in your ipython_config.py file if it
-            works for you.  Test it and turn it on permanently if it works with
-            your system.  The magic function %%color_info allows you to toggle this
-            inter- actively for testing.""")
-        paa('--no-color-info',
-            action='store_false', dest='InteractiveShell.color_info',
-            help="Disable using colors for info related things.")
-        paa('--confirm-exit',
-            action='store_true', dest='TerminalInteractiveShell.confirm_exit',
-            help=
-            """Set to confirm when you try to exit IPython with an EOF (Control-D
-            in Unix, Control-Z/Enter in Windows). By typing 'exit', 'quit' or
-            '%%Exit', you can force a direct exit without any confirmation.""")
-        paa('--no-confirm-exit',
-            action='store_false', dest='TerminalInteractiveShell.confirm_exit',
-            help="Don't prompt the user when exiting.")
-        paa('--deep-reload',
-            action='store_true', dest='InteractiveShell.deep_reload',
-            help=
-            """Enable deep (recursive) reloading by default. IPython can use the
-            deep_reload module which reloads changes in modules recursively (it
-            replaces the reload() function, so you don't need to change anything to
-            use it). deep_reload() forces a full reload of modules whose code may
-            have changed, which the default reload() function does not.  When
-            deep_reload is off, IPython will use the normal reload(), but
-            deep_reload will still be available as dreload(). This fea- ture is off
-            by default [which means that you have both normal reload() and
-            dreload()].""")
-        paa('--no-deep-reload',
-            action='store_false', dest='InteractiveShell.deep_reload',
-            help="Disable deep (recursive) reloading by default.")
-        paa('--editor',
-            type=str, dest='TerminalInteractiveShell.editor',
-            help="Set the editor used by IPython (default to $EDITOR/vi/notepad).",
-            metavar='TerminalInteractiveShell.editor')
-        paa('--log','-l', 
-            action='store_true', dest='InteractiveShell.logstart',
-            help="Start logging to the default log file (./ipython_log.py).")
-        paa('--logfile','-lf',
-            type=unicode, dest='InteractiveShell.logfile',
-            help="Start logging to logfile with this name.",
-            metavar='InteractiveShell.logfile')
-        paa('--log-append','-la',
-            type=unicode, dest='InteractiveShell.logappend',
-            help="Start logging to the given file in append mode.",
-            metavar='InteractiveShell.logfile')
-        paa('--pdb',
-            action='store_true', dest='InteractiveShell.pdb',
-            help="Enable auto calling the pdb debugger after every exception.")
-        paa('--no-pdb',
-            action='store_false', dest='InteractiveShell.pdb',
-            help="Disable auto calling the pdb debugger after every exception.")
-        paa('--pprint',
-            action='store_true', dest='PlainTextFormatter.pprint',
-            help="Enable auto pretty printing of results.")
-        paa('--no-pprint',
-            action='store_false', dest='PlainTextFormatter.pprint',
-            help="Disable auto auto pretty printing of results.")
-        paa('--prompt-in1','-pi1',
-            type=str, dest='InteractiveShell.prompt_in1',
-            help=
-            """Set the main input prompt ('In [\#]: ').  Note that if you are using
-            numbered prompts, the number is represented with a '\#' in the string.
-            Don't forget to quote strings with spaces embedded in them. Most
-            bash-like escapes can be used to customize IPython's prompts, as well
-            as a few additional ones which are IPython-spe- cific.  All valid
-            prompt escapes are described in detail in the Customization section of
-            the IPython manual.""",
-            metavar='InteractiveShell.prompt_in1')
-        paa('--prompt-in2','-pi2',
-            type=str, dest='InteractiveShell.prompt_in2',
-            help=
-            """Set the secondary input prompt (' .\D.: ').  Similar to the previous
-            option, but used for the continuation prompts. The special sequence
-            '\D' is similar to '\#', but with all digits replaced by dots (so you
-            can have your continuation prompt aligned with your input prompt).
-            Default: ' .\D.: ' (note three spaces at the start for alignment with
-            'In [\#]')""",
-            metavar='InteractiveShell.prompt_in2')
-        paa('--prompt-out','-po',
-            type=str, dest='InteractiveShell.prompt_out',
-            help="Set the output prompt ('Out[\#]:')",
-            metavar='InteractiveShell.prompt_out')
-        paa('--quick',
-            action='store_true', dest='Global.quick',
-            help="Enable quick startup with no config files.")
-        paa('--readline',
-            action='store_true', dest='InteractiveShell.readline_use',
-            help="Enable readline for command line usage.")
-        paa('--no-readline',
-            action='store_false', dest='InteractiveShell.readline_use',
-            help="Disable readline for command line usage.")
-        paa('--screen-length','-sl',
-            type=int, dest='TerminalInteractiveShell.screen_length',
-            help=
-            """Number of lines of your screen, used to control printing of very
-            long strings.  Strings longer than this number of lines will be sent
-            through a pager instead of directly printed.  The default value for
-            this is 0, which means IPython will auto-detect your screen size every
-            time it needs to print certain potentially long strings (this doesn't
-            change the behavior of the 'print' keyword, it's only triggered
-            internally). If for some reason this isn't working well (it needs
-            curses support), specify it yourself. Otherwise don't change the
-            default.""",
-            metavar='TerminalInteractiveShell.screen_length')
-        paa('--separate-in','-si',
-            type=str, dest='InteractiveShell.separate_in',
-            help="Separator before input prompts.  Default '\\n'.",
-            metavar='InteractiveShell.separate_in')
-        paa('--separate-out','-so',
-            type=str, dest='InteractiveShell.separate_out',
-            help="Separator before output prompts.  Default 0 (nothing).",
-            metavar='InteractiveShell.separate_out')
-        paa('--separate-out2','-so2', 
-            type=str, dest='InteractiveShell.separate_out2',
-            help="Separator after output prompts.  Default 0 (nonight).",
-            metavar='InteractiveShell.separate_out2')
-        paa('--no-sep',
-            action='store_true', dest='Global.nosep',
-            help="Eliminate all spacing between prompts.")
-        paa('--term-title',
-            action='store_true', dest='TerminalInteractiveShell.term_title',
-            help="Enable auto setting the terminal title.")
-        paa('--no-term-title',
-            action='store_false', dest='TerminalInteractiveShell.term_title',
-            help="Disable auto setting the terminal title.")
-        paa('--xmode',
-            type=str, dest='InteractiveShell.xmode',
-            help=
-            """Exception reporting mode ('Plain','Context','Verbose').  Plain:
-            similar to python's normal traceback printing.  Context: prints 5 lines
-            of context source code around each line in the traceback.  Verbose:
-            similar to Context, but additionally prints the variables currently
-            visible where the exception happened (shortening their strings if too
-            long).  This can potentially be very slow, if you happen to have a huge
-            data structure whose string representation is complex to compute.
-            Your computer may appear to freeze for a while with cpu usage at 100%%.
-            If this occurs, you can cancel the traceback with Ctrl-C (maybe hitting
-            it more than once).
-            """,
-            metavar='InteractiveShell.xmode')
-        paa('--ext',
-            type=str, dest='Global.extra_extension',
-            help="The dotted module name of an IPython extension to load.",
-            metavar='Global.extra_extension')
-        paa('-c',
-            type=str, dest='Global.code_to_run',
-            help="Execute the given command string.",
-            metavar='Global.code_to_run')
-        paa('-i',
-            action='store_true', dest='Global.force_interact',
-            help=
-            "If running code from the command line, become interactive afterwards.")
-
-        # Options to start with GUI control enabled from the beginning
-        paa('--gui',
-            type=str, dest='Global.gui',
-            help="Enable GUI event loop integration ('qt', 'wx', 'gtk').",
-            metavar='gui-mode')
-        paa('--pylab','-pylab',
-            type=str, dest='Global.pylab',
-            nargs='?', const='auto', metavar='gui-mode',
-            help="Pre-load matplotlib and numpy for interactive use. "+
-            "If no value is given, the gui backend is matplotlib's, else use "+
-            "one of:  ['tk', 'qt', 'wx', 'gtk', 'osx'].")
-
-        # Legacy GUI options.  Leave them in for backwards compatibility, but the
-        # 'thread' names are really a misnomer now.
-        paa('--wthread', '-wthread',
-            action='store_true', dest='Global.wthread',
-            help=
-            """Enable wxPython event loop integration. (DEPRECATED, use --gui wx)""")
-        paa('--q4thread', '--qthread', '-q4thread', '-qthread',
-            action='store_true', dest='Global.q4thread',
-            help=
-            """Enable Qt4 event loop integration. Qt3 is no longer supported.
-            (DEPRECATED, use --gui qt)""")
-        paa('--gthread', '-gthread',
-            action='store_true', dest='Global.gthread',
-            help=
-            """Enable GTK event loop integration. (DEPRECATED, use --gui gtk)""")
-
-
 #-----------------------------------------------------------------------------
 # Crash handler for this application
 #-----------------------------------------------------------------------------
@@ -377,271 +117,219 b' class IPAppCrashHandler(CrashHandler):'
 
         return ''.join(report)
 
+#-----------------------------------------------------------------------------
+# Aliases and Flags
+#-----------------------------------------------------------------------------
+flags = dict(base_flags)
+flags.update(shell_flags)
+addflag = lambda *args: flags.update(boolean_flag(*args))
+addflag('autoedit-syntax', 'TerminalInteractiveShell.autoedit_syntax',
+        'Turn on auto editing of files with syntax errors.',
+        'Turn off auto editing of files with syntax errors.'
+)
+addflag('banner', 'TerminalIPythonApp.display_banner',
+        "Display a banner upon starting IPython.",
+        "Don't display a banner upon starting IPython."
+)
+addflag('confirm-exit', 'TerminalInteractiveShell.confirm_exit',
+    """Set to confirm when you try to exit IPython with an EOF (Control-D
+    in Unix, Control-Z/Enter in Windows). By typing 'exit' or 'quit',
+    you can force a direct exit without any confirmation.""",
+    "Don't prompt the user when exiting."
+)
+addflag('term-title', 'TerminalInteractiveShell.term_title',
+    "Enable auto setting the terminal title.",
+    "Disable auto setting the terminal title."
+)
+classic_config = Config()
+classic_config.InteractiveShell.cache_size = 0
+classic_config.PlainTextFormatter.pprint = False
+classic_config.InteractiveShell.prompt_in1 = '>>> '
+classic_config.InteractiveShell.prompt_in2 = '... '
+classic_config.InteractiveShell.prompt_out = ''
+classic_config.InteractiveShell.separate_in = ''
+classic_config.InteractiveShell.separate_out = ''
+classic_config.InteractiveShell.separate_out2 = ''
+classic_config.InteractiveShell.colors = 'NoColor'
+classic_config.InteractiveShell.xmode = 'Plain'
+
+flags['classic']=(
+    classic_config,
+    "Gives IPython a similar feel to the classic Python prompt."
+)
+# # log doesn't make so much sense this way anymore
+# paa('--log','-l',
+#     action='store_true', dest='InteractiveShell.logstart',
+#     help="Start logging to the default log file (./ipython_log.py).")
+#
+# # quick is harder to implement
+flags['quick']=(
+    {'TerminalIPythonApp' : {'quick' : True}},
+    "Enable quick startup with no config files."
+)
+
+flags['i'] = (
+    {'TerminalIPythonApp' : {'force_interact' : True}},
+    "If running code from the command line, become interactive afterwards."
+)
+flags['pylab'] = (
+    {'TerminalIPythonApp' : {'pylab' : 'auto'}},
+    """Pre-load matplotlib and numpy for interactive use with
+    the default matplotlib backend."""
+)
+
+aliases = dict(base_aliases)
+aliases.update(shell_aliases)
+
+# it's possible we don't want short aliases for *all* of these:
+aliases.update(dict(
+    gui='TerminalIPythonApp.gui',
+    pylab='TerminalIPythonApp.pylab',
+))
 
 #-----------------------------------------------------------------------------
 # Main classes and functions
 #-----------------------------------------------------------------------------
 
-class IPythonApp(Application):
+class TerminalIPythonApp(BaseIPythonApplication, InteractiveShellApp):
     name = u'ipython'
-    #: argparse formats better the 'usage' than the 'description' field
-    description = None
-    usage = usage.cl_usage
-    command_line_loader = IPAppConfigLoader
+    description = usage.cl_usage
     default_config_file_name = default_config_file_name
     crash_handler_class = IPAppCrashHandler
+    
+    flags = Dict(flags)
+    aliases = Dict(aliases)
+    classes = [InteractiveShellApp, TerminalInteractiveShell, ProfileDir, PlainTextFormatter]
+    subcommands = Dict(dict(
+        qtconsole=('IPython.frontend.qt.console.qtconsoleapp.IPythonQtConsoleApp',
+            """Launch the IPython Qt Console."""
+        ),
+        profile = ("IPython.core.profileapp.ProfileApp",
+            "Create and manage IPython profiles.")
+    ))
+    
+    # *do* autocreate requested profile, but don't create the config file.
+    auto_create=Bool(True)
+    # configurables
+    ignore_old_config=Bool(False, config=True,
+        help="Suppress warning messages about legacy config files"
+    )
+    quick = Bool(False, config=True,
+        help="""Start IPython quickly by skipping the loading of config files."""
+    )
+    def _quick_changed(self, name, old, new):
+        if new:
+            self.load_config_file = lambda *a, **kw: None
+            self.ignore_old_config=True
+
+    gui = CaselessStrEnum(('qt','wx','gtk'), config=True,
+        help="Enable GUI event loop integration ('qt', 'wx', 'gtk')."
+    )
+    pylab = CaselessStrEnum(['tk', 'qt', 'wx', 'gtk', 'osx', 'auto'],
+        config=True,
+        help="""Pre-load matplotlib and numpy for interactive use,
+        selecting a particular matplotlib backend and loop integration.
+        """
+    )
+    display_banner = Bool(True, config=True,
+        help="Whether to display a banner upon starting IPython."
+    )
+
+    # if there is code of files to run from the cmd line, don't interact
+    # unless the --i flag (App.force_interact) is true.
+    force_interact = Bool(False, config=True,
+        help="""If a command or file is given via the command-line,
+        e.g. 'ipython foo.py"""
+    )
+    def _force_interact_changed(self, name, old, new):
+        if new:
+            self.interact = True
+    
+    def _file_to_run_changed(self, name, old, new):
+        if new and not self.force_interact:
+                self.interact = False
+    _code_to_run_changed = _file_to_run_changed
+
+    # internal, not-configurable
+    interact=Bool(True)
+
+
+    def initialize(self, argv=None):
+        """Do actions after construct, but before starting the app."""
+        super(TerminalIPythonApp, self).initialize(argv)
+        if self.subapp is not None:
+            # don't bother initializing further, starting subapp
+            return
+        if not self.ignore_old_config:
+            check_for_old_config(self.ipython_dir)
+        # print self.extra_args
+        if self.extra_args:
+            self.file_to_run = self.extra_args[0]
+        # create the shell
+        self.init_shell()
+        # and draw the banner
+        self.init_banner()
+        # Now a variety of things that happen after the banner is printed.
+        self.init_gui_pylab()
+        self.init_extensions()
+        self.init_code()
 
-    def create_default_config(self):
-        super(IPythonApp, self).create_default_config()
-        # Eliminate multiple lookups
-        Global = self.default_config.Global
-
-        # Set all default values
-        Global.display_banner = True
-        
-        # If the -c flag is given or a file is given to run at the cmd line
-        # like "ipython foo.py", normally we exit without starting the main
-        # loop.  The force_interact config variable allows a user to override
-        # this and interact.  It is also set by the -i cmd line flag, just
-        # like Python.
-        Global.force_interact = False
-
-        # By default always interact by starting the IPython mainloop.
-        Global.interact = True
-
-        # No GUI integration by default
-        Global.gui = False
-        # Pylab off by default
-        Global.pylab = False
-
-        # Deprecated versions of gui support that used threading, we support
-        # them just for bacwards compatibility as an alternate spelling for
-        # '--gui X'
-        Global.qthread = False
-        Global.q4thread = False
-        Global.wthread = False
-        Global.gthread = False
-
-    def load_file_config(self):
-        if hasattr(self.command_line_config.Global, 'quick'):
-            if self.command_line_config.Global.quick:
-                self.file_config = Config()
-                return
-        super(IPythonApp, self).load_file_config()
-
-    def post_load_file_config(self):
-        if hasattr(self.command_line_config.Global, 'extra_extension'):
-            if not hasattr(self.file_config.Global, 'extensions'):
-                self.file_config.Global.extensions = []
-            self.file_config.Global.extensions.append(
-                self.command_line_config.Global.extra_extension)
-            del self.command_line_config.Global.extra_extension
-
-    def pre_construct(self):
-        config = self.master_config
-
-        if hasattr(config.Global, 'classic'):
-            if config.Global.classic:
-                config.InteractiveShell.cache_size = 0
-                config.PlainTextFormatter.pprint = False
-                config.InteractiveShell.prompt_in1 = '>>> '
-                config.InteractiveShell.prompt_in2 = '... '
-                config.InteractiveShell.prompt_out = ''
-                config.InteractiveShell.separate_in = \
-                    config.InteractiveShell.separate_out = \
-                    config.InteractiveShell.separate_out2 = ''
-                config.InteractiveShell.colors = 'NoColor'
-                config.InteractiveShell.xmode = 'Plain'
-
-        if hasattr(config.Global, 'nosep'):
-            if config.Global.nosep:
-                config.InteractiveShell.separate_in = \
-                config.InteractiveShell.separate_out = \
-                config.InteractiveShell.separate_out2 = ''
-
-        # if there is code of files to run from the cmd line, don't interact
-        # unless the -i flag (Global.force_interact) is true.
-        code_to_run = config.Global.get('code_to_run','')
-        file_to_run = False
-        if self.extra_args and self.extra_args[0]:
-                file_to_run = True
-        if file_to_run or code_to_run:
-            if not config.Global.force_interact:
-                config.Global.interact = False
-
-    def construct(self):
+    def init_shell(self):
+        """initialize the InteractiveShell instance"""
         # I am a little hesitant to put these into InteractiveShell itself.
         # But that might be the place for them
         sys.path.insert(0, '')
 
         # Create an InteractiveShell instance.
-        self.shell = TerminalInteractiveShell.instance(config=self.master_config)
-
-    def post_construct(self):
-        """Do actions after construct, but before starting the app."""
-        config = self.master_config
-        
         # shell.display_banner should always be False for the terminal 
         # based app, because we call shell.show_banner() by hand below
         # so the banner shows *before* all extension loading stuff.
-        self.shell.display_banner = False
-        if config.Global.display_banner and \
-            config.Global.interact:
-            self.shell.show_banner()
+        self.shell = TerminalInteractiveShell.instance(config=self.config,
+                        display_banner=False, profile_dir=self.profile_dir,
+                        ipython_dir=self.ipython_dir)
 
+    def init_banner(self):
+        """optionally display the banner"""
+        if self.display_banner and self.interact:
+            self.shell.show_banner()
         # Make sure there is a space below the banner.
         if self.log_level <= logging.INFO: print
 
-        # Now a variety of things that happen after the banner is printed.
-        self._enable_gui_pylab()
-        self._load_extensions()
-        self._run_exec_lines()
-        self._run_exec_files()
-        self._run_cmd_line_code()
 
-    def _enable_gui_pylab(self):
+    def init_gui_pylab(self):
         """Enable GUI event loop integration, taking pylab into account."""
-        Global = self.master_config.Global
-
-        # Select which gui to use
-        if Global.gui:
-            gui = Global.gui
-        # The following are deprecated, but there's likely to be a lot of use
-        # of this form out there, so we might as well support it for now.  But
-        # the --gui option above takes precedence.
-        elif Global.wthread:
-            gui = inputhook.GUI_WX
-        elif Global.qthread:
-            gui = inputhook.GUI_QT
-        elif Global.gthread:
-            gui = inputhook.GUI_GTK
-        else:
-            gui = None
+        gui = self.gui
 
-        # Using --pylab will also require gui activation, though which toolkit
+        # Using `pylab` will also require gui activation, though which toolkit
         # to use may be chosen automatically based on mpl configuration.
-        if Global.pylab:
+        if self.pylab:
             activate = self.shell.enable_pylab
-            if Global.pylab == 'auto':
+            if self.pylab == 'auto':
                 gui = None
             else:
-                gui = Global.pylab
+                gui = self.pylab
         else:
             # Enable only GUI integration, no pylab
             activate = inputhook.enable_gui
 
-        if gui or Global.pylab:
+        if gui or self.pylab:
             try:
                 self.log.info("Enabling GUI event loop integration, "
-                              "toolkit=%s, pylab=%s" % (gui, Global.pylab) )
+                              "toolkit=%s, pylab=%s" % (gui, self.pylab) )
                 activate(gui)
             except:
                 self.log.warn("Error in enabling GUI event loop integration:")
                 self.shell.showtraceback()
 
-    def _load_extensions(self):
-        """Load all IPython extensions in Global.extensions.
-
-        This uses the :meth:`ExtensionManager.load_extensions` to load all
-        the extensions listed in ``self.master_config.Global.extensions``.
-        """
-        try:
-            if hasattr(self.master_config.Global, 'extensions'):
-                self.log.debug("Loading IPython extensions...")
-                extensions = self.master_config.Global.extensions
-                for ext in extensions:
-                    try:
-                        self.log.info("Loading IPython extension: %s" % ext)                        
-                        self.shell.extension_manager.load_extension(ext)
-                    except:
-                        self.log.warn("Error in loading extension: %s" % ext)
-                        self.shell.showtraceback()
-        except:
-            self.log.warn("Unknown error in loading extensions:")
-            self.shell.showtraceback()
-
-    def _run_exec_lines(self):
-        """Run lines of code in Global.exec_lines in the user's namespace."""
-        try:
-            if hasattr(self.master_config.Global, 'exec_lines'):
-                self.log.debug("Running code from Global.exec_lines...")
-                exec_lines = self.master_config.Global.exec_lines
-                for line in exec_lines:
-                    try:
-                        self.log.info("Running code in user namespace: %s" %
-                                      line)
-                        self.shell.run_cell(line, store_history=False)
-                    except:
-                        self.log.warn("Error in executing line in user "
-                                      "namespace: %s" % line)
-                        self.shell.showtraceback()
-        except:
-            self.log.warn("Unknown error in handling Global.exec_lines:")
-            self.shell.showtraceback()
-
-    def _exec_file(self, fname):
-        full_filename = filefind(fname, [u'.', self.ipython_dir])
-        if os.path.isfile(full_filename):
-            if full_filename.endswith(u'.py'):
-                self.log.info("Running file in user namespace: %s" %
-                              full_filename)
-                # Ensure that __file__ is always defined to match Python behavior
-                self.shell.user_ns['__file__'] = fname
-                try:
-                    self.shell.safe_execfile(full_filename, self.shell.user_ns)
-                finally:
-                    del self.shell.user_ns['__file__']
-            elif full_filename.endswith('.ipy'):
-                self.log.info("Running file in user namespace: %s" %
-                              full_filename)
-                self.shell.safe_execfile_ipy(full_filename)
-            else:
-                self.log.warn("File does not have a .py or .ipy extension: <%s>"
-                               % full_filename)
-    def _run_exec_files(self):
-        try:
-            if hasattr(self.master_config.Global, 'exec_files'):
-                self.log.debug("Running files in Global.exec_files...")
-                exec_files = self.master_config.Global.exec_files
-                for fname in exec_files:
-                    self._exec_file(fname)
-        except:
-            self.log.warn("Unknown error in handling Global.exec_files:")
-            self.shell.showtraceback()
-
-    def _run_cmd_line_code(self):
-        if hasattr(self.master_config.Global, 'code_to_run'):
-            line = self.master_config.Global.code_to_run
-            try:
-                self.log.info("Running code given at command line (-c): %s" %
-                              line)
-                self.shell.run_cell(line, store_history=False)
-            except:
-                self.log.warn("Error in executing line in user namespace: %s" %
-                              line)
-                self.shell.showtraceback()
-            return
-        # Like Python itself, ignore the second if the first of these is present
-        try:
-            fname = self.extra_args[0]
-        except:
-            pass
-        else:
-            try:
-                self._exec_file(fname)
-            except:
-                self.log.warn("Error in executing file in user namespace: %s" %
-                              fname)
-                self.shell.showtraceback()
-
-    def start_app(self):
-        if not getattr(self.master_config.Global, 'ignore_old_config', False):
-            check_for_old_config(self.ipython_dir)
-        if self.master_config.Global.interact:
+    def start(self):
+        if self.subapp is not None:
+            return self.subapp.start()
+        # perform any prexec steps:
+        if self.interact:
             self.log.debug("Starting IPython's mainloop...")
             self.shell.mainloop()
         else:
-            self.log.debug("IPython not interactive, start_app is no-op...")
+            self.log.debug("IPython not interactive...")
 
 
 def load_default_config(ipython_dir=None):
@@ -651,16 +339,19 b' def load_default_config(ipython_dir=None):'
     """
     if ipython_dir is None:
         ipython_dir = get_ipython_dir()
-    cl = PyFileConfigLoader(default_config_file_name, ipython_dir)
+    profile_dir = os.path.join(ipython_dir, 'profile_default')
+    cl = PyFileConfigLoader(default_config_file_name, profile_dir)
     config = cl.load_config()
     return config
 
 
 def launch_new_instance():
     """Create and run a full blown IPython instance"""
-    app = IPythonApp()
+    app = TerminalIPythonApp.instance()
+    app.initialize()
     app.start()
 
 
 if __name__ == '__main__':
     launch_new_instance()
+

IPython/lib/irunner.py

@@ -304,9 +304,7 b' class IPythonRunner(InteractiveRunner):'
     def __init__(self,program = 'ipython',args=None,out=sys.stdout,echo=True):
         """New runner, optionally passing the ipython command to use."""
         
-        args0 = ['--colors','NoColor',
-                 '-pi1','In [\\#]: ',
-                 '-pi2','   .\\D.: ',
+        args0 = ['colors=NoColor',
                  '--no-term-title',
                  '--no-autoindent']
         if args is None: args = args0

IPython/lib/pylabtools.py

@@ -87,8 +87,8 b' def figsize(sizex, sizey):'
     matplotlib.rcParams['figure.figsize'] = [sizex, sizey]
 
 
-def figure_to_svg(fig):
-    """Convert a figure to svg for inline display."""
+def print_figure(fig, fmt='png'):
+    """Convert a figure to svg or png for inline display."""
     # When there's an empty figure, we shouldn't return anything, otherwise we
     # get big blank areas in the qt console.
     if not fig.axes:
@@ -100,12 +100,13 b' def figure_to_svg(fig):'
     fig.set_edgecolor('white')
     try:
         string_io = StringIO()
-        fig.canvas.print_figure(string_io, format='svg')
-        svg = string_io.getvalue()
+        # use 72 dpi to match QTConsole's dpi
+        fig.canvas.print_figure(string_io, format=fmt, dpi=72)
+        data = string_io.getvalue()
     finally:
         fig.set_facecolor(fc)
         fig.set_edgecolor(ec)
-    return svg
+    return data
 
 
 # We need a little factory function here to create the closure where
@@ -150,6 +151,29 b' def mpl_runner(safe_execfile):'
     return mpl_execfile
 
 
+def select_figure_format(shell, fmt):
+    """Select figure format for inline backend, either 'png' or 'svg'.
+
+    Using this method ensures only one figure format is active at a time.
+    """
+    from matplotlib.figure import Figure
+    from IPython.zmq.pylab import backend_inline
+
+    svg_formatter = shell.display_formatter.formatters['image/svg+xml']
+    png_formatter = shell.display_formatter.formatters['image/png']
+
+    if fmt=='png':
+        svg_formatter.type_printers.pop(Figure, None)
+        png_formatter.for_type(Figure, lambda fig: print_figure(fig, 'png'))
+    elif fmt=='svg':
+        png_formatter.type_printers.pop(Figure, None)
+        svg_formatter.for_type(Figure, lambda fig: print_figure(fig, 'svg'))
+    else:
+        raise ValueError("supported formats are: 'png', 'svg', not %r"%fmt)
+
+    # set the format to be used in the backend()
+    backend_inline._figure_format = fmt
+
 #-----------------------------------------------------------------------------
 # Code for initializing matplotlib and importing pylab
 #-----------------------------------------------------------------------------
@@ -208,7 +232,6 b' def activate_matplotlib(backend):'
     # For this, we wrap it into a decorator which adds a 'called' flag.
     pylab.draw_if_interactive = flag_calls(pylab.draw_if_interactive)
 
-
 def import_pylab(user_ns, backend, import_all=True, shell=None):
     """Import the standard pylab symbols into user_ns."""
 
@@ -228,43 +251,32 b' def import_pylab(user_ns, backend, import_all=True, shell=None):'
         # If using our svg payload backend, register the post-execution
         # function that will pick up the results for display.  This can only be
         # done with access to the real shell object.
+        #
+        from IPython.zmq.pylab.backend_inline import InlineBackendConfig
+
+        cfg = InlineBackendConfig.instance(config=shell.config)
+        cfg.shell = shell
+
         if backend == backends['inline']:
-            from IPython.zmq.pylab.backend_inline import flush_svg
+            from IPython.zmq.pylab.backend_inline import flush_figures
             from matplotlib import pyplot
-            shell.register_post_execute(flush_svg)
-            # The typical default figure size is too large for inline use,
-            # so we shrink the figure size to 6x4, and tweak fonts to
-            # make that fit.  This is configurable via Global.pylab_inline_rc,
-            # or rather it will be once the zmq kernel is hooked up to
-            # the config system.
-            
-            default_rc = {
-                'figure.figsize': (6.0,4.0),
-                # 12pt labels get cutoff on 6x4 logplots, so use 10pt.
-                'font.size': 10,
-                # 10pt still needs a little more room on the xlabel:
-                'figure.subplot.bottom' : .125
-            }
-            rc = getattr(shell.config.Global, 'pylab_inline_rc', default_rc)
-            pyplot.rcParams.update(rc)
-            shell.config.Global.pylab_inline_rc = rc
+            shell.register_post_execute(flush_figures)
+            # load inline_rc
+            pyplot.rcParams.update(cfg.rc)
             
             # Add 'figsize' to pyplot and to the user's namespace
             user_ns['figsize'] = pyplot.figsize = figsize
             shell.user_ns_hidden['figsize'] = figsize
         
+        # Setup the default figure format
+        fmt = cfg.figure_format
+        select_figure_format(shell, fmt)
+
         # The old pastefig function has been replaced by display
-        # Always add this svg formatter so display works.
-        from IPython.core.display import display, display_svg
-        svg_formatter = shell.display_formatter.formatters['image/svg+xml']
-        svg_formatter.for_type_by_name(
-            'matplotlib.figure','Figure',figure_to_svg
-        )
+        from IPython.core.display import display
         # Add display and display_png to the user's namespace
         user_ns['display'] = display
         shell.user_ns_hidden['display'] = display
-        user_ns['display_svg'] = display_svg
-        shell.user_ns_hidden['display_svg'] = display_svg
         user_ns['getfigs'] = getfigs
         shell.user_ns_hidden['getfigs'] = getfigs
 

IPython/lib/tests/test_pylabtools.py

@@ -41,7 +41,7 b' from .. import pylabtools as pt'
 def test_figure_to_svg():
     # simple empty-figure test
     fig = plt.figure()
-    yield nt.assert_equal(pt.figure_to_svg(fig), None)
+    yield nt.assert_equal(pt.print_figure(fig, 'svg'), None)
 
     plt.close('all')
 
@@ -50,5 +50,5 b' def test_figure_to_svg():'
     ax = fig.add_subplot(1,1,1)
     ax.plot([1,2,3])
     plt.draw()
-    svg = pt.figure_to_svg(fig)[:100].lower()
+    svg = pt.print_figure(fig, 'svg')[:100].lower()
     yield nt.assert_true('doctype svg' in svg)

IPython/parallel/__init__.py

@@ -1,4 +1,9 b''
-"""The IPython ZMQ-based parallel computing interface."""
+"""The IPython ZMQ-based parallel computing interface.
+
+Authors:
+
+* MinRK
+"""
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2011 The IPython Development Team
 #

IPython/parallel/apps/ipclusterapp.py

@@ -2,10 +2,16 b''
 # encoding: utf-8
 """
 The ipcluster application.
+
+Authors:
+
+* Brian Granger
+* MinRK
+
 """
 
 #-----------------------------------------------------------------------------
-#  Copyright (C) 2008-2009  The IPython Development Team
+#  Copyright (C) 2008-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
@@ -25,12 +31,18 b' from subprocess import check_call, CalledProcessError, PIPE'
 import zmq
 from zmq.eventloop import ioloop
 
-from IPython.external.argparse import ArgumentParser, SUPPRESS
+from IPython.config.application import Application, boolean_flag
+from IPython.config.loader import Config
+from IPython.core.application import BaseIPythonApplication
+from IPython.core.profiledir import ProfileDir
+from IPython.utils.daemonize import daemonize
 from IPython.utils.importstring import import_item
+from IPython.utils.traitlets import Int, Unicode, Bool, CFloat, Dict, List
 
-from IPython.parallel.apps.clusterdir import (
-    ApplicationWithClusterDir, ClusterDirConfigLoader,
-    ClusterDirError, PIDFileError
+from IPython.parallel.apps.baseapp import (
+    BaseParallelApplication,
+    PIDFileError,
+    base_flags, base_aliases
 )
 
 
@@ -42,16 +54,15 b' from IPython.parallel.apps.clusterdir import ('
 default_config_file_name = u'ipcluster_config.py'
 
 
-_description = """\
-Start an IPython cluster for parallel computing.\n\n
+_description = """Start an IPython cluster for parallel computing.
 
 An IPython cluster consists of 1 controller and 1 or more engines.
 This command automates the startup of these processes using a wide
 range of startup methods (SSH, local processes, PBS, mpiexec,
 Windows HPC Server 2008). To start a cluster with 4 engines on your
-local host simply do 'ipcluster start -n 4'. For more complex usage 
-you will typically do 'ipcluster create -p mycluster', then edit
-configuration files, followed by 'ipcluster start -p mycluster -n 4'.
+local host simply do 'ipcluster start n=4'. For more complex usage
+you will typically do 'ipcluster create profile=mycluster', then edit
+configuration files, followed by 'ipcluster start profile=mycluster n=4'.
 """
 
 
@@ -72,426 +83,286 b' NO_CLUSTER = 12'
 
 
 #-----------------------------------------------------------------------------
-# Command line options
+# Main application
 #-----------------------------------------------------------------------------
+start_help = """Start an IPython cluster for parallel computing
+
+Start an ipython cluster by its profile name or cluster
+directory. Cluster directories contain configuration, log and
+security related files and are named using the convention
+'profile_<name>' and should be creating using the 'start'
+subcommand of 'ipcluster'. If your cluster directory is in 
+the cwd or the ipython directory, you can simply refer to it
+using its profile name, 'ipcluster start n=4 profile=<profile>`,
+otherwise use the 'profile_dir' option.
+"""
+stop_help = """Stop a running IPython cluster
+
+Stop a running ipython cluster by its profile name or cluster
+directory. Cluster directories are named using the convention
+'profile_<name>'. If your cluster directory is in 
+the cwd or the ipython directory, you can simply refer to it
+using its profile name, 'ipcluster stop profile=<profile>`, otherwise
+use the 'profile_dir' option.
+"""
+engines_help = """Start engines connected to an existing IPython cluster
+
+Start one or more engines to connect to an existing Cluster
+by profile name or cluster directory.
+Cluster directories contain configuration, log and
+security related files and are named using the convention
+'profile_<name>' and should be creating using the 'start'
+subcommand of 'ipcluster'. If your cluster directory is in 
+the cwd or the ipython directory, you can simply refer to it
+using its profile name, 'ipcluster engines n=4 profile=<profile>`,
+otherwise use the 'profile_dir' option.
+"""
+stop_aliases = dict(
+    signal='IPClusterStop.signal',
+    profile='BaseIPythonApplication.profile',
+    profile_dir='ProfileDir.location',
+)
 
-
-class IPClusterAppConfigLoader(ClusterDirConfigLoader):
-
-    def _add_arguments(self):
-        # Don't call ClusterDirConfigLoader._add_arguments as we don't want
-        # its defaults on self.parser. Instead, we will put those on
-        # default options on our subparsers.
-        
-        # This has all the common options that all subcommands use
-        parent_parser1 = ArgumentParser(
-            add_help=False,
-            argument_default=SUPPRESS
-        )
-        self._add_ipython_dir(parent_parser1)
-        self._add_log_level(parent_parser1)
-
-        # This has all the common options that other subcommands use
-        parent_parser2 = ArgumentParser(
-            add_help=False,
-            argument_default=SUPPRESS
-        )
-        self._add_cluster_profile(parent_parser2)
-        self._add_cluster_dir(parent_parser2)
-        self._add_work_dir(parent_parser2)
-        paa = parent_parser2.add_argument
-        paa('--log-to-file',
-            action='store_true', dest='Global.log_to_file', 
-            help='Log to a file in the log directory (default is stdout)')
-
-        # Create the object used to create the subparsers.
-        subparsers = self.parser.add_subparsers(
-            dest='Global.subcommand',
-            title='ipcluster subcommands',
-            description=
-            """ipcluster has a variety of subcommands. The general way of 
-            running ipcluster is 'ipcluster <cmd> [options]'. To get help
-            on a particular subcommand do 'ipcluster <cmd> -h'."""
-            # help="For more help, type 'ipcluster <cmd> -h'",
-        )
-
-        # The "list" subcommand parser
-        parser_list = subparsers.add_parser(
-            'list',
-            parents=[parent_parser1],
-            argument_default=SUPPRESS,
-            help="List all clusters in cwd and ipython_dir.",
-            description=
-            """List all available clusters, by cluster directory, that can
-            be found in the current working directly or in the ipython
-            directory. Cluster directories are named using the convention
-            'cluster_<profile>'."""
-        )
-
-        # The "create" subcommand parser
-        parser_create = subparsers.add_parser(
-            'create',
-            parents=[parent_parser1, parent_parser2],
-            argument_default=SUPPRESS,
-            help="Create a new cluster directory.",
-            description=
-            """Create an ipython cluster directory by its profile name or 
-            cluster directory path. Cluster directories contain 
-            configuration, log and security related files and are named 
-            using the convention 'cluster_<profile>'. By default they are
-            located in your ipython directory. Once created, you will
-            probably need to edit the configuration files in the cluster
-            directory to configure your cluster. Most users will create a
-            cluster directory by profile name, 
-            'ipcluster create -p mycluster', which will put the directory
-            in '<ipython_dir>/cluster_mycluster'. 
-            """
-        )
-        paa = parser_create.add_argument
-        paa('--reset-config',
-            dest='Global.reset_config', action='store_true',
-            help=
-            """Recopy the default config files to the cluster directory.
-            You will loose any modifications you have made to these files.""")
-
-        # The "start" subcommand parser
-        parser_start = subparsers.add_parser(
-            'start',
-            parents=[parent_parser1, parent_parser2],
-            argument_default=SUPPRESS,
-            help="Start a cluster.",
-            description=
-            """Start an ipython cluster by its profile name or cluster 
-            directory. Cluster directories contain configuration, log and
-            security related files and are named using the convention
-            'cluster_<profile>' and should be creating using the 'start'
-            subcommand of 'ipcluster'. If your cluster directory is in 
-            the cwd or the ipython directory, you can simply refer to it
-            using its profile name, 'ipcluster start -n 4 -p <profile>`,
-            otherwise use the '--cluster-dir' option.
-            """
-        )
+class IPClusterStop(BaseParallelApplication):
+    name = u'ipcluster'
+    description = stop_help
+    config_file_name = Unicode(default_config_file_name)
+    
+    signal = Int(signal.SIGINT, config=True,
+        help="signal to use for stopping processes.")
         
-        paa = parser_start.add_argument
-        paa('-n', '--number',
-            type=int, dest='Global.n',
-            help='The number of engines to start.',
-            metavar='Global.n')
-        paa('--clean-logs',
-            dest='Global.clean_logs', action='store_true',
-            help='Delete old log flies before starting.')
-        paa('--no-clean-logs',
-            dest='Global.clean_logs', action='store_false',
-            help="Don't delete old log flies before starting.")
-        paa('--daemon',
-            dest='Global.daemonize', action='store_true',
-            help='Daemonize the ipcluster program. This implies --log-to-file')
-        paa('--no-daemon',
-            dest='Global.daemonize', action='store_false',
-            help="Dont't daemonize the ipcluster program.")
-        paa('--delay',
-            type=float, dest='Global.delay',
-            help="Specify the delay (in seconds) between starting the controller and starting the engine(s).")
-
-        # The "stop" subcommand parser
-        parser_stop = subparsers.add_parser(
-            'stop',
-            parents=[parent_parser1, parent_parser2],
-            argument_default=SUPPRESS,
-            help="Stop a running cluster.",
-            description=
-            """Stop a running ipython cluster by its profile name or cluster 
-            directory. Cluster directories are named using the convention
-            'cluster_<profile>'. If your cluster directory is in 
-            the cwd or the ipython directory, you can simply refer to it
-            using its profile name, 'ipcluster stop -p <profile>`, otherwise
-            use the '--cluster-dir' option.
-            """
-        )
-        paa = parser_stop.add_argument
-        paa('--signal',
-            dest='Global.signal', type=int,
-            help="The signal number to use in stopping the cluster (default=2).",
-            metavar="Global.signal")
+    aliases = Dict(stop_aliases)
+    
+    def start(self):
+        """Start the app for the stop subcommand."""
+        try:
+            pid = self.get_pid_from_file()
+        except PIDFileError:
+            self.log.critical(
+                'Could not read pid file, cluster is probably not running.'
+            )
+            # Here I exit with a unusual exit status that other processes
+            # can watch for to learn how I existed.
+            self.remove_pid_file()
+            self.exit(ALREADY_STOPPED)
         
-        # the "engines" subcommand parser
-        parser_engines = subparsers.add_parser(
-            'engines',
-            parents=[parent_parser1, parent_parser2],
-            argument_default=SUPPRESS,
-            help="Attach some engines to an existing controller or cluster.",
-            description=
-            """Start one or more engines to connect to an existing Cluster
-            by profile name or cluster directory.
-            Cluster directories contain configuration, log and
-            security related files and are named using the convention
-            'cluster_<profile>' and should be creating using the 'start'
-            subcommand of 'ipcluster'. If your cluster directory is in 
-            the cwd or the ipython directory, you can simply refer to it
-            using its profile name, 'ipcluster engines -n 4 -p <profile>`,
-            otherwise use the '--cluster-dir' option.
-            """
-        )
-        paa = parser_engines.add_argument
-        paa('-n', '--number',
-            type=int, dest='Global.n',
-            help='The number of engines to start.',
-            metavar='Global.n')
-        paa('--daemon',
-            dest='Global.daemonize', action='store_true',
-            help='Daemonize the ipcluster program. This implies --log-to-file')
-        paa('--no-daemon',
-            dest='Global.daemonize', action='store_false',
-            help="Dont't daemonize the ipcluster program.")
-
-#-----------------------------------------------------------------------------
-# Main application
-#-----------------------------------------------------------------------------
-
-
-class IPClusterApp(ApplicationWithClusterDir):
+        if not self.check_pid(pid):
+            self.log.critical(
+                'Cluster [pid=%r] is not running.' % pid
+            )
+            self.remove_pid_file()
+            # Here I exit with a unusual exit status that other processes
+            # can watch for to learn how I existed.
+            self.exit(ALREADY_STOPPED)
+            
+        elif os.name=='posix':
+            sig = self.signal
+            self.log.info(
+                "Stopping cluster [pid=%r] with [signal=%r]" % (pid, sig)
+            )
+            try:
+                os.kill(pid, sig)
+            except OSError:
+                self.log.error("Stopping cluster failed, assuming already dead.",
+                    exc_info=True)
+                self.remove_pid_file()
+        elif os.name=='nt':
+            try:
+                # kill the whole tree
+                p = check_call(['taskkill', '-pid', str(pid), '-t', '-f'], stdout=PIPE,stderr=PIPE)
+            except (CalledProcessError, OSError):
+                self.log.error("Stopping cluster failed, assuming already dead.",
+                    exc_info=True)
+            self.remove_pid_file()
+    
+engine_aliases = {}
+engine_aliases.update(base_aliases)
+engine_aliases.update(dict(
+    n='IPClusterEngines.n',
+    elauncher = 'IPClusterEngines.engine_launcher_class',
+))
+class IPClusterEngines(BaseParallelApplication):
 
     name = u'ipcluster'
-    description = _description
+    description = engines_help
     usage = None
-    command_line_loader = IPClusterAppConfigLoader
-    default_config_file_name = default_config_file_name
+    config_file_name = Unicode(default_config_file_name)
     default_log_level = logging.INFO
-    auto_create_cluster_dir = False
-
-    def create_default_config(self):
-        super(IPClusterApp, self).create_default_config()
-        self.default_config.Global.controller_launcher = \
-            'IPython.parallel.apps.launcher.LocalControllerLauncher'
-        self.default_config.Global.engine_launcher = \
-            'IPython.parallel.apps.launcher.LocalEngineSetLauncher'
-        self.default_config.Global.n = 2
-        self.default_config.Global.delay = 2
-        self.default_config.Global.reset_config = False
-        self.default_config.Global.clean_logs = True
-        self.default_config.Global.signal = signal.SIGINT
-        self.default_config.Global.daemonize = False
-
-    def find_resources(self):
-        subcommand = self.command_line_config.Global.subcommand
-        if subcommand=='list':
-            self.list_cluster_dirs()
-            # Exit immediately because there is nothing left to do.
-            self.exit()
-        elif subcommand=='create':
-            self.auto_create_cluster_dir = True
-            super(IPClusterApp, self).find_resources()
-        elif subcommand=='start' or subcommand=='stop':
-            self.auto_create_cluster_dir = True
-            try:
-                super(IPClusterApp, self).find_resources()
-            except ClusterDirError:
-                raise ClusterDirError(
-                    "Could not find a cluster directory. A cluster dir must "
-                    "be created before running 'ipcluster start'.  Do "
-                    "'ipcluster create -h' or 'ipcluster list -h' for more "
-                    "information about creating and listing cluster dirs."
-                )
-        elif subcommand=='engines':
-            self.auto_create_cluster_dir = False
-            try:
-                super(IPClusterApp, self).find_resources()
-            except ClusterDirError:
-                raise ClusterDirError(
-                    "Could not find a cluster directory. A cluster dir must "
-                    "be created before running 'ipcluster start'.  Do "
-                    "'ipcluster create -h' or 'ipcluster list -h' for more "
-                    "information about creating and listing cluster dirs."
-                )
+    classes = List()
+    def _classes_default(self):
+        from IPython.parallel.apps import launcher
+        launchers = launcher.all_launchers
+        eslaunchers = [ l for l in launchers if 'EngineSet' in l.__name__]
+        return [ProfileDir]+eslaunchers
+    
+    n = Int(2, config=True,
+        help="The number of engines to start.")
 
-    def list_cluster_dirs(self):
-        # Find the search paths
-        cluster_dir_paths = os.environ.get('IPCLUSTER_DIR_PATH','')
-        if cluster_dir_paths:
-            cluster_dir_paths = cluster_dir_paths.split(':')
-        else:
-            cluster_dir_paths = []
-        try:
-            ipython_dir = self.command_line_config.Global.ipython_dir
-        except AttributeError:
-            ipython_dir = self.default_config.Global.ipython_dir
-        paths = [os.getcwd(), ipython_dir] + \
-            cluster_dir_paths
-        paths = list(set(paths))
-
-        self.log.info('Searching for cluster dirs in paths: %r' % paths)
-        for path in paths:
-            files = os.listdir(path)
-            for f in files:
-                full_path = os.path.join(path, f)
-                if os.path.isdir(full_path) and f.startswith('cluster_'):
-                    profile = full_path.split('_')[-1]
-                    start_cmd = 'ipcluster start -p %s -n 4' % profile
-                    print start_cmd + " ==> " + full_path
-
-    def pre_construct(self):
-        # IPClusterApp.pre_construct() is where we cd to the working directory.
-        super(IPClusterApp, self).pre_construct()
-        config = self.master_config
-        try:
-            daemon = config.Global.daemonize
-            if daemon:
-                config.Global.log_to_file = True
-        except AttributeError:
-            pass
+    engine_launcher_class = Unicode('LocalEngineSetLauncher',
+        config=True,
+        help="The class for launching a set of Engines."
+        )
+    daemonize = Bool(False, config=True,
+        help='Daemonize the ipcluster program. This implies --log-to-file')
 
-    def construct(self):
-        config = self.master_config
-        subcmd = config.Global.subcommand
-        reset = config.Global.reset_config
-        if subcmd == 'list':
-            return
-        if subcmd == 'create':
-            self.log.info('Copying default config files to cluster directory '
-            '[overwrite=%r]' % (reset,))
-            self.cluster_dir_obj.copy_all_config_files(overwrite=reset)
-        if subcmd =='start':
-            self.cluster_dir_obj.copy_all_config_files(overwrite=False)
-            self.start_logging()
-            self.loop = ioloop.IOLoop.instance()
-            # reactor.callWhenRunning(self.start_launchers)
-            dc = ioloop.DelayedCallback(self.start_launchers, 0, self.loop)
-            dc.start()
-        if subcmd == 'engines':
-            self.start_logging()
-            self.loop = ioloop.IOLoop.instance()
-            # reactor.callWhenRunning(self.start_launchers)
-            engine_only = lambda : self.start_launchers(controller=False)
-            dc = ioloop.DelayedCallback(engine_only, 0, self.loop)
-            dc.start()
+    def _daemonize_changed(self, name, old, new):
+        if new:
+            self.log_to_file = True
 
-    def start_launchers(self, controller=True):
-        config = self.master_config
-
-        # Create the launchers. In both bases, we set the work_dir of
-        # the launcher to the cluster_dir. This is where the launcher's
-        # subprocesses will be launched. It is not where the controller
-        # and engine will be launched.
-        if controller:
-            cl_class = import_item(config.Global.controller_launcher)
-            self.controller_launcher = cl_class(
-                work_dir=self.cluster_dir, config=config,
-                logname=self.log.name
-            )
-            # Setup the observing of stopping. If the controller dies, shut
-            # everything down as that will be completely fatal for the engines.
-            self.controller_launcher.on_stop(self.stop_launchers)
-            # But, we don't monitor the stopping of engines. An engine dying
-            # is just fine and in principle a user could start a new engine.
-            # Also, if we did monitor engine stopping, it is difficult to
-            # know what to do when only some engines die. Currently, the
-            # observing of engine stopping is inconsistent. Some launchers
-            # might trigger on a single engine stopping, other wait until
-            # all stop.  TODO: think more about how to handle this.
-        else:
-            self.controller_launcher = None
-        
-        el_class = import_item(config.Global.engine_launcher)
-        self.engine_launcher = el_class(
-            work_dir=self.cluster_dir, config=config, logname=self.log.name
-        )
+    aliases = Dict(engine_aliases)
+    # flags = Dict(flags)
+    _stopping = False
 
+    def initialize(self, argv=None):
+        super(IPClusterEngines, self).initialize(argv)
+        self.init_signal()
+        self.init_launchers()
+    
+    def init_launchers(self):
+        self.engine_launcher = self.build_launcher(self.engine_launcher_class)
+        self.engine_launcher.on_stop(lambda r: self.loop.stop())
+    
+    def init_signal(self):
         # Setup signals
         signal.signal(signal.SIGINT, self.sigint_handler)
-
-        # Start the controller and engines
-        self._stopping = False  # Make sure stop_launchers is not called 2x.
-        if controller:
-            self.start_controller()
-        dc = ioloop.DelayedCallback(self.start_engines, 1000*config.Global.delay*controller, self.loop)
-        dc.start()
-        self.startup_message()
-
-    def startup_message(self, r=None):
-        self.log.info("IPython cluster: started")
-        return r
-        
-    def start_controller(self, r=None):
-        # self.log.info("In start_controller")
-        config = self.master_config
-        d = self.controller_launcher.start(
-            cluster_dir=config.Global.cluster_dir
+    
+    def build_launcher(self, clsname):
+        """import and instantiate a Launcher based on importstring"""
+        if '.' not in clsname:
+            # not a module, presume it's the raw name in apps.launcher
+            clsname = 'IPython.parallel.apps.launcher.'+clsname
+        # print repr(clsname)
+        klass = import_item(clsname)
+
+        launcher = klass(
+            work_dir=self.profile_dir.location, config=self.config, log=self.log
         )
-        return d
-            
-    def start_engines(self, r=None):
-        # self.log.info("In start_engines")
-        config = self.master_config
-        
-        d = self.engine_launcher.start(
-            config.Global.n,
-            cluster_dir=config.Global.cluster_dir
+        return launcher
+    
+    def start_engines(self):
+        self.log.info("Starting %i engines"%self.n)
+        self.engine_launcher.start(
+            self.n,
+            self.profile_dir.location
         )
-        return d
-
-    def stop_controller(self, r=None):
-        # self.log.info("In stop_controller")
-        if self.controller_launcher and self.controller_launcher.running:
-            return self.controller_launcher.stop()
 
-    def stop_engines(self, r=None):
-        # self.log.info("In stop_engines")
+    def stop_engines(self):
+        self.log.info("Stopping Engines...")
         if self.engine_launcher.running:
             d = self.engine_launcher.stop()
-            # d.addErrback(self.log_err)
             return d
         else:
             return None
 
-    def log_err(self, f):
-        self.log.error(f.getTraceback())
-        return None
-        
     def stop_launchers(self, r=None):
         if not self._stopping:
             self._stopping = True
-            # if isinstance(r, failure.Failure):
-            #     self.log.error('Unexpected error in ipcluster:')
-            #     self.log.info(r.getTraceback())
             self.log.error("IPython cluster: stopping")
-            # These return deferreds. We are not doing anything with them
-            # but we are holding refs to them as a reminder that they 
-            # do return deferreds.
-            d1 = self.stop_engines()
-            d2 = self.stop_controller()
+            self.stop_engines()
             # Wait a few seconds to let things shut down.
             dc = ioloop.DelayedCallback(self.loop.stop, 4000, self.loop)
             dc.start()
-            # reactor.callLater(4.0, reactor.stop)
 
     def sigint_handler(self, signum, frame):
+        self.log.debug("SIGINT received, stopping launchers...")
         self.stop_launchers()
         
     def start_logging(self):
         # Remove old log files of the controller and engine
-        if self.master_config.Global.clean_logs:
-            log_dir = self.master_config.Global.log_dir
+        if self.clean_logs:
+            log_dir = self.profile_dir.log_dir
             for f in os.listdir(log_dir):
                 if re.match(r'ip(engine|controller)z-\d+\.(log|err|out)',f):
                     os.remove(os.path.join(log_dir, f))
         # This will remove old log files for ipcluster itself
-        super(IPClusterApp, self).start_logging()
-
-    def start_app(self):
-        """Start the application, depending on what subcommand is used."""
-        subcmd = self.master_config.Global.subcommand
-        if subcmd=='create' or subcmd=='list':
-            return
-        elif subcmd=='start':
-            self.start_app_start()
-        elif subcmd=='stop':
-            self.start_app_stop()
-        elif subcmd=='engines':
-            self.start_app_engines()
-
-    def start_app_start(self):
+        # super(IPBaseParallelApplication, self).start_logging()
+
+    def start(self):
+        """Start the app for the engines subcommand."""
+        self.log.info("IPython cluster: started")
+        # First see if the cluster is already running
+        
+        # Now log and daemonize
+        self.log.info(
+            'Starting engines with [daemon=%r]' % self.daemonize
+        )
+        # TODO: Get daemonize working on Windows or as a Windows Server.
+        if self.daemonize:
+            if os.name=='posix':
+                daemonize()
+
+        dc = ioloop.DelayedCallback(self.start_engines, 0, self.loop)
+        dc.start()
+        # Now write the new pid file AFTER our new forked pid is active.
+        # self.write_pid_file()
+        try:
+            self.loop.start()
+        except KeyboardInterrupt:
+            pass
+        except zmq.ZMQError as e:
+            if e.errno == errno.EINTR:
+                pass
+            else:
+                raise
+
+start_aliases = {}
+start_aliases.update(engine_aliases)
+start_aliases.update(dict(
+    delay='IPClusterStart.delay',
+    clean_logs='IPClusterStart.clean_logs',
+))
+
+class IPClusterStart(IPClusterEngines):
+
+    name = u'ipcluster'
+    description = start_help
+    default_log_level = logging.INFO
+    auto_create = Bool(True, config=True,
+        help="whether to create the profile_dir if it doesn't exist")
+    classes = List()
+    def _classes_default(self,):
+        from IPython.parallel.apps import launcher
+        return [ProfileDir] + [IPClusterEngines] + launcher.all_launchers
+    
+    clean_logs = Bool(True, config=True, 
+        help="whether to cleanup old logs before starting")
+
+    delay = CFloat(1., config=True,
+        help="delay (in s) between starting the controller and the engines")
+
+    controller_launcher_class = Unicode('LocalControllerLauncher',
+        config=True,
+        help="The class for launching a Controller."
+        )
+    reset = Bool(False, config=True,
+        help="Whether to reset config files as part of '--create'."
+        )
+    
+    # flags = Dict(flags)
+    aliases = Dict(start_aliases)
+
+    def init_launchers(self):
+        self.controller_launcher = self.build_launcher(self.controller_launcher_class)
+        self.engine_launcher = self.build_launcher(self.engine_launcher_class)
+        self.controller_launcher.on_stop(self.stop_launchers)
+    
+    def start_controller(self):
+        self.controller_launcher.start(
+            self.profile_dir.location
+        )
+        
+    def stop_controller(self):
+        # self.log.info("In stop_controller")
+        if self.controller_launcher and self.controller_launcher.running:
+            return self.controller_launcher.stop()
+
+    def stop_launchers(self, r=None):
+        if not self._stopping:
+            self.stop_controller()
+            super(IPClusterStart, self).stop_launchers()
+
+    def start(self):
         """Start the app for the start subcommand."""
-        config = self.master_config
         # First see if the cluster is already running
         try:
             pid = self.get_pid_from_file()
@@ -512,14 +383,17 b' class IPClusterApp(ApplicationWithClusterDir):'
 
         # Now log and daemonize
         self.log.info(
-            'Starting ipcluster with [daemon=%r]' % config.Global.daemonize
+            'Starting ipcluster with [daemon=%r]' % self.daemonize
         )
         # TODO: Get daemonize working on Windows or as a Windows Server.
-        if config.Global.daemonize:
+        if self.daemonize:
             if os.name=='posix':
-                from twisted.scripts._twistd_unix import daemonize
                 daemonize()
 
+        dc = ioloop.DelayedCallback(self.start_controller, 0, self.loop)
+        dc.start()
+        dc = ioloop.DelayedCallback(self.start_engines, 1000*self.delay, self.loop)
+        dc.start()
         # Now write the new pid file AFTER our new forked pid is active.
         self.write_pid_file()
         try:
@@ -534,81 +408,36 b' class IPClusterApp(ApplicationWithClusterDir):'
         finally:
             self.remove_pid_file()
 
-    def start_app_engines(self):
-        """Start the app for the start subcommand."""
-        config = self.master_config
-        # First see if the cluster is already running
-        
-        # Now log and daemonize
-        self.log.info(
-            'Starting engines with [daemon=%r]' % config.Global.daemonize
-        )
-        # TODO: Get daemonize working on Windows or as a Windows Server.
-        if config.Global.daemonize:
-            if os.name=='posix':
-                from twisted.scripts._twistd_unix import daemonize
-                daemonize()
+base='IPython.parallel.apps.ipclusterapp.IPCluster'
 
-        # Now write the new pid file AFTER our new forked pid is active.
-        # self.write_pid_file()
-        try:
-            self.loop.start()
-        except KeyboardInterrupt:
-            pass
-        except zmq.ZMQError as e:
-            if e.errno == errno.EINTR:
-                pass
-            else:
-                raise
-        # self.remove_pid_file()
+class IPClusterApp(Application):
+    name = u'ipcluster'
+    description = _description
     
-    def start_app_stop(self):
-        """Start the app for the stop subcommand."""
-        config = self.master_config
-        try:
-            pid = self.get_pid_from_file()
-        except PIDFileError:
-            self.log.critical(
-                'Could not read pid file, cluster is probably not running.'
-            )
-            # Here I exit with a unusual exit status that other processes
-            # can watch for to learn how I existed.
-            self.remove_pid_file()
-            self.exit(ALREADY_STOPPED)
-        
-        if not self.check_pid(pid):
-            self.log.critical(
-                'Cluster [pid=%r] is not running.' % pid
-            )
-            self.remove_pid_file()
-            # Here I exit with a unusual exit status that other processes
-            # can watch for to learn how I existed.
-            self.exit(ALREADY_STOPPED)
-            
-        elif os.name=='posix':
-            sig = config.Global.signal
-            self.log.info(
-                "Stopping cluster [pid=%r] with [signal=%r]" % (pid, sig)
-            )
-            try:
-                os.kill(pid, sig)
-            except OSError:
-                self.log.error("Stopping cluster failed, assuming already dead.",
-                    exc_info=True)
-                self.remove_pid_file()
-        elif os.name=='nt':
-            try:
-                # kill the whole tree
-                p = check_call(['taskkill', '-pid', str(pid), '-t', '-f'], stdout=PIPE,stderr=PIPE)
-            except (CalledProcessError, OSError):
-                self.log.error("Stopping cluster failed, assuming already dead.",
-                    exc_info=True)
-            self.remove_pid_file()
-
+    subcommands = {
+                'start' : (base+'Start', start_help),
+                'stop' : (base+'Stop', stop_help),
+                'engines' : (base+'Engines', engines_help),
+    }
+    
+    # no aliases or flags for parent App
+    aliases = Dict()
+    flags = Dict()
+    
+    def start(self):
+        if self.subapp is None:
+            print "No subcommand specified. Must specify one of: %s"%(self.subcommands.keys())
+            print
+            self.print_description()
+            self.print_subcommands()
+            self.exit(1)
+        else:
+            return self.subapp.start()
 
 def launch_new_instance():
     """Create and run the IPython cluster."""
-    app = IPClusterApp()
+    app = IPClusterApp.instance()
+    app.initialize()
     app.start()
 
 

IPython/parallel/apps/ipcontrollerapp.py

@@ -2,10 +2,16 b''
 # encoding: utf-8
 """
 The IPython controller application.
+
+Authors:
+
+* Brian Granger
+* MinRK
+
 """
 
 #-----------------------------------------------------------------------------
-#  Copyright (C) 2008-2009  The IPython Development Team
+#  Copyright (C) 2008-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
@@ -17,31 +23,46 b' The IPython controller application.'
 
 from __future__ import with_statement
 
-import copy
 import os
-import logging
 import socket
 import stat
 import sys
 import uuid
 
+from multiprocessing import Process
+
 import zmq
+from zmq.devices import ProcessMonitoredQueue
 from zmq.log.handlers import PUBHandler
 from zmq.utils import jsonapi as json
 
-from IPython.config.loader import Config
-
-from IPython.parallel import factory
+from IPython.config.application import boolean_flag
+from IPython.core.profiledir import ProfileDir
 
-from IPython.parallel.apps.clusterdir import (
-    ApplicationWithClusterDir,
-    ClusterDirConfigLoader
+from IPython.parallel.apps.baseapp import (
+    BaseParallelApplication,
+    base_flags
 )
-from IPython.parallel.util import disambiguate_ip_address, split_url
-# from IPython.kernel.fcutil import FCServiceFactory, FURLError
-from IPython.utils.traitlets import Instance, Unicode
+from IPython.utils.importstring import import_item
+from IPython.utils.traitlets import Instance, Unicode, Bool, List, Dict
+
+# from IPython.parallel.controller.controller import ControllerFactory
+from IPython.zmq.session import Session
+from IPython.parallel.controller.heartmonitor import HeartMonitor
+from IPython.parallel.controller.hub import HubFactory
+from IPython.parallel.controller.scheduler import TaskScheduler,launch_scheduler
+from IPython.parallel.controller.sqlitedb import SQLiteDB
 
-from IPython.parallel.controller.controller import ControllerFactory
+from IPython.parallel.util import signal_children, split_url
+
+# conditional import of MongoDB backend class
+
+try:
+    from IPython.parallel.controller.mongodb import MongoDB
+except ImportError:
+    maybe_mongo = []
+else:
+    maybe_mongo = [MongoDB]
 
 
 #-----------------------------------------------------------------------------
@@ -59,238 +80,112 b' The IPython controller provides a gateway between the IPython engines and'
 clients. The controller needs to be started before the engines and can be
 configured using command line options or using a cluster directory. Cluster
 directories contain config, log and security files and are usually located in
-your ipython directory and named as "cluster_<profile>". See the --profile
-and --cluster-dir options for details.
+your ipython directory and named as "profile_name". See the `profile`
+and `profile_dir` options for details.
 """
 
-#-----------------------------------------------------------------------------
-# Default interfaces
-#-----------------------------------------------------------------------------
-
-# The default client interfaces for FCClientServiceFactory.interfaces
-default_client_interfaces = Config()
-default_client_interfaces.Default.url_file = 'ipcontroller-client.url'
-
-# Make this a dict we can pass to Config.__init__ for the default
-default_client_interfaces = dict(copy.deepcopy(default_client_interfaces.items()))
 
 
 
-# The default engine interfaces for FCEngineServiceFactory.interfaces
-default_engine_interfaces = Config()
-default_engine_interfaces.Default.url_file = u'ipcontroller-engine.url'
-
-# Make this a dict we can pass to Config.__init__ for the default
-default_engine_interfaces = dict(copy.deepcopy(default_engine_interfaces.items()))
-
-
-#-----------------------------------------------------------------------------
-# Service factories
-#-----------------------------------------------------------------------------
-
-# 
-# class FCClientServiceFactory(FCServiceFactory):
-#     """A Foolscap implementation of the client services."""
-# 
-#     cert_file = Unicode(u'ipcontroller-client.pem', config=True)
-#     interfaces = Instance(klass=Config, kw=default_client_interfaces,
-#                           allow_none=False, config=True)
-# 
-# 
-# class FCEngineServiceFactory(FCServiceFactory):
-#     """A Foolscap implementation of the engine services."""
-# 
-#     cert_file = Unicode(u'ipcontroller-engine.pem', config=True)
-#     interfaces = Instance(klass=dict, kw=default_engine_interfaces,
-#                           allow_none=False, config=True)
-# 
-
-#-----------------------------------------------------------------------------
-# Command line options
-#-----------------------------------------------------------------------------
-
-
-class IPControllerAppConfigLoader(ClusterDirConfigLoader):
-
-    def _add_arguments(self):
-        super(IPControllerAppConfigLoader, self)._add_arguments()
-        paa = self.parser.add_argument
-        
-        ## Hub Config:
-        paa('--mongodb', 
-            dest='HubFactory.db_class', action='store_const',
-            const='IPython.parallel.controller.mongodb.MongoDB', 
-            help='Use MongoDB for task storage [default: in-memory]')
-        paa('--sqlite', 
-            dest='HubFactory.db_class', action='store_const',
-            const='IPython.parallel.controller.sqlitedb.SQLiteDB',
-            help='Use SQLite3 for DB task storage [default: in-memory]')
-        paa('--hb',
-            type=int, dest='HubFactory.hb', nargs=2,
-            help='The (2) ports the Hub\'s Heartmonitor will use for the heartbeat '
-            'connections [default: random]',
-            metavar='Hub.hb_ports')
-        paa('--ping',
-            type=int, dest='HubFactory.ping',
-            help='The frequency at which the Hub pings the engines for heartbeats '
-            ' (in ms) [default: 100]',
-            metavar='Hub.ping')
-        
-        # Client config
-        paa('--client-ip',
-            type=str, dest='HubFactory.client_ip', 
-            help='The IP address or hostname the Hub will listen on for '
-            'client connections. Both engine-ip and client-ip can be set simultaneously '
-            'via --ip [default: loopback]',
-            metavar='Hub.client_ip')
-        paa('--client-transport',
-            type=str, dest='HubFactory.client_transport', 
-            help='The ZeroMQ transport the Hub will use for '
-            'client connections. Both engine-transport and client-transport can be set simultaneously '
-            'via --transport [default: tcp]',
-            metavar='Hub.client_transport')
-        paa('--query',
-            type=int, dest='HubFactory.query_port', 
-            help='The port on which the Hub XREP socket will listen for result queries from clients [default: random]',
-            metavar='Hub.query_port')
-        paa('--notifier',
-            type=int, dest='HubFactory.notifier_port', 
-            help='The port on which the Hub PUB socket will listen for notification connections [default: random]',
-            metavar='Hub.notifier_port')
-        
-        # Engine config
-        paa('--engine-ip',
-            type=str, dest='HubFactory.engine_ip', 
-            help='The IP address or hostname the Hub will listen on for '
-            'engine connections. This applies to the Hub and its schedulers'
-            'engine-ip and client-ip can be set simultaneously '
-            'via --ip [default: loopback]',
-            metavar='Hub.engine_ip')
-        paa('--engine-transport',
-            type=str, dest='HubFactory.engine_transport', 
-            help='The ZeroMQ transport the Hub will use for '
-            'client connections. Both engine-transport and client-transport can be set simultaneously '
-            'via --transport [default: tcp]',
-            metavar='Hub.engine_transport')
-        
-        # Scheduler config
-        paa('--mux',
-            type=int, dest='ControllerFactory.mux', nargs=2,
-            help='The (2) ports the MUX scheduler will listen on for client,engine '
-            'connections, respectively [default: random]',
-            metavar='Scheduler.mux_ports')
-        paa('--task',
-            type=int, dest='ControllerFactory.task', nargs=2,
-            help='The (2) ports the Task scheduler will listen on for client,engine '
-            'connections, respectively [default: random]',
-            metavar='Scheduler.task_ports')
-        paa('--control',
-            type=int, dest='ControllerFactory.control', nargs=2,
-            help='The (2) ports the Control scheduler will listen on for client,engine '
-            'connections, respectively [default: random]',
-            metavar='Scheduler.control_ports')
-        paa('--iopub',
-            type=int, dest='ControllerFactory.iopub', nargs=2,
-            help='The (2) ports the IOPub scheduler will listen on for client,engine '
-            'connections, respectively [default: random]',
-            metavar='Scheduler.iopub_ports')
-            
-        paa('--scheme',
-            type=str, dest='HubFactory.scheme',
-            choices = ['pure', 'lru', 'plainrandom', 'weighted', 'twobin','leastload'],
-            help='select the task scheduler scheme  [default: Python LRU]',
-            metavar='Scheduler.scheme')
-        paa('--usethreads',
-            dest='ControllerFactory.usethreads', action="store_true",
-            help='Use threads instead of processes for the schedulers',
-            )
-        paa('--hwm',
-            dest='TaskScheduler.hwm', type=int,
-            help='specify the High Water Mark (HWM) '
-            'in the Python scheduler. This is the maximum number '
-            'of allowed outstanding tasks on each engine.',
-            )
-        
-        ## Global config
-        paa('--log-to-file',
-            action='store_true', dest='Global.log_to_file',
-            help='Log to a file in the log directory (default is stdout)')
-        paa('--log-url',
-            type=str, dest='Global.log_url',
-            help='Broadcast logs to an iploggerz process [default: disabled]')
-        paa('-r','--reuse-files', 
-            action='store_true', dest='Global.reuse_files',
-            help='Try to reuse existing json connection files.')
-        paa('--no-secure',
-            action='store_false', dest='Global.secure',
-            help='Turn off execution keys (default).')
-        paa('--secure',
-            action='store_true', dest='Global.secure',
-            help='Turn on execution keys.')
-        paa('--execkey',
-            type=str, dest='Global.exec_key',
-            help='path to a file containing an execution key.',
-            metavar='keyfile')
-        paa('--ssh',
-            type=str, dest='Global.sshserver',
-            help='ssh url for clients to use when connecting to the Controller '
-            'processes. It should be of the form: [user@]server[:port]. The '
-            'Controller\'s listening addresses must be accessible from the ssh server',
-            metavar='Global.sshserver')
-        paa('--location',
-            type=str, dest='Global.location',
-            help="The external IP or domain name of this machine, used for disambiguating "
-            "engine and client connections.",
-            metavar='Global.location')
-        factory.add_session_arguments(self.parser)
-        factory.add_registration_arguments(self.parser)
-
-
 #-----------------------------------------------------------------------------
 # The main application
 #-----------------------------------------------------------------------------
-
-
-class IPControllerApp(ApplicationWithClusterDir):
+flags = {}
+flags.update(base_flags)
+flags.update({
+    'usethreads' : ( {'IPControllerApp' : {'use_threads' : True}},
+                    'Use threads instead of processes for the schedulers'),
+    'sqlitedb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.sqlitedb.SQLiteDB'}},
+                    'use the SQLiteDB backend'),
+    'mongodb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.mongodb.MongoDB'}},
+                    'use the MongoDB backend'),
+    'dictdb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.dictdb.DictDB'}},
+                    'use the in-memory DictDB backend'),
+    'reuse' : ({'IPControllerApp' : {'reuse_files' : True}},
+                    'reuse existing json connection files')
+})
+
+flags.update(boolean_flag('secure', 'IPControllerApp.secure',
+    "Use HMAC digests for authentication of messages.",
+    "Don't authenticate messages."
+))
+
+class IPControllerApp(BaseParallelApplication):
 
     name = u'ipcontroller'
     description = _description
-    command_line_loader = IPControllerAppConfigLoader
-    default_config_file_name = default_config_file_name
-    auto_create_cluster_dir = True
+    config_file_name = Unicode(default_config_file_name)
+    classes = [ProfileDir, Session, HubFactory, TaskScheduler, HeartMonitor, SQLiteDB] + maybe_mongo
     
-
-    def create_default_config(self):
-        super(IPControllerApp, self).create_default_config()
-        # Don't set defaults for Global.secure or Global.reuse_furls
-        # as those are set in a component.
-        self.default_config.Global.import_statements = []
-        self.default_config.Global.clean_logs = True
-        self.default_config.Global.secure = True
-        self.default_config.Global.reuse_files = False
-        self.default_config.Global.exec_key = "exec_key.key"
-        self.default_config.Global.sshserver = None
-        self.default_config.Global.location = None
-
-    def pre_construct(self):
-        super(IPControllerApp, self).pre_construct()
-        c = self.master_config
-        # The defaults for these are set in FCClientServiceFactory and
-        # FCEngineServiceFactory, so we only set them here if the global
-        # options have be set to override the class level defaults.
+    # change default to True
+    auto_create = Bool(True, config=True,
+        help="""Whether to create profile dir if it doesn't exist.""")
+    
+    reuse_files = Bool(False, config=True,
+        help='Whether to reuse existing json connection files.'
+    )
+    secure = Bool(True, config=True,
+        help='Whether to use HMAC digests for extra message authentication.'
+    )
+    ssh_server = Unicode(u'', config=True,
+        help="""ssh url for clients to use when connecting to the Controller
+        processes. It should be of the form: [user@]server[:port]. The
+        Controller's listening addresses must be accessible from the ssh server""",
+    )
+    location = Unicode(u'', config=True,
+        help="""The external IP or domain name of the Controller, used for disambiguating
+        engine and client connections.""",
+    )
+    import_statements = List([], config=True,
+        help="import statements to be run at startup.  Necessary in some environments"
+    )
+
+    use_threads = Bool(False, config=True,
+        help='Use threads instead of processes for the schedulers',
+        )
+
+    # internal
+    children = List()
+    mq_class = Unicode('zmq.devices.ProcessMonitoredQueue')
+
+    def _use_threads_changed(self, name, old, new):
+        self.mq_class = 'zmq.devices.%sMonitoredQueue'%('Thread' if new else 'Process')
+
+    aliases = Dict(dict(
+        log_level = 'IPControllerApp.log_level',
+        log_url = 'IPControllerApp.log_url',
+        reuse_files = 'IPControllerApp.reuse_files',
+        secure = 'IPControllerApp.secure',
+        ssh = 'IPControllerApp.ssh_server',
+        use_threads = 'IPControllerApp.use_threads',
+        import_statements = 'IPControllerApp.import_statements',
+        location = 'IPControllerApp.location',
+
+        ident = 'Session.session',
+        user = 'Session.username',
+        exec_key = 'Session.keyfile',
+
+        url = 'HubFactory.url',
+        ip = 'HubFactory.ip',
+        transport = 'HubFactory.transport',
+        port = 'HubFactory.regport',
+
+        ping = 'HeartMonitor.period',
+
+        scheme = 'TaskScheduler.scheme_name',
+        hwm = 'TaskScheduler.hwm',
+
+
+        profile = "BaseIPythonApplication.profile",
+        profile_dir = 'ProfileDir.location',
         
-        # if hasattr(c.Global, 'reuse_furls'):
-        #     c.FCClientServiceFactory.reuse_furls = c.Global.reuse_furls
-        #     c.FCEngineServiceFactory.reuse_furls = c.Global.reuse_furls
-        #     del c.Global.reuse_furls
-        # if hasattr(c.Global, 'secure'):
-        #     c.FCClientServiceFactory.secure = c.Global.secure
-        #     c.FCEngineServiceFactory.secure = c.Global.secure
-        #     del c.Global.secure
+    ))
+    flags = Dict(flags)
     
+
     def save_connection_dict(self, fname, cdict):
         """save a connection dict to json file."""
-        c = self.master_config
+        c = self.config
         url = cdict['url']
         location = cdict['location']
         if not location:
@@ -301,43 +196,41 b' class IPControllerApp(ApplicationWithClusterDir):'
             else:
                 location = socket.gethostbyname_ex(socket.gethostname())[2][-1]
             cdict['location'] = location
-        fname = os.path.join(c.Global.security_dir, fname)
+        fname = os.path.join(self.profile_dir.security_dir, fname)
         with open(fname, 'w') as f:
             f.write(json.dumps(cdict, indent=2))
         os.chmod(fname, stat.S_IRUSR|stat.S_IWUSR)
     
     def load_config_from_json(self):
         """load config from existing json connector files."""
-        c = self.master_config
+        c = self.config
         # load from engine config
-        with open(os.path.join(c.Global.security_dir, 'ipcontroller-engine.json')) as f:
+        with open(os.path.join(self.profile_dir.security_dir, 'ipcontroller-engine.json')) as f:
             cfg = json.loads(f.read())
-        key = c.SessionFactory.exec_key = cfg['exec_key']
+        key = c.Session.key = cfg['exec_key']
         xport,addr = cfg['url'].split('://')
         c.HubFactory.engine_transport = xport
         ip,ports = addr.split(':')
         c.HubFactory.engine_ip = ip
         c.HubFactory.regport = int(ports)
-        c.Global.location = cfg['location']
+        self.location = cfg['location']
         
         # load client config
-        with open(os.path.join(c.Global.security_dir, 'ipcontroller-client.json')) as f:
+        with open(os.path.join(self.profile_dir.security_dir, 'ipcontroller-client.json')) as f:
             cfg = json.loads(f.read())
         assert key == cfg['exec_key'], "exec_key mismatch between engine and client keys"
         xport,addr = cfg['url'].split('://')
         c.HubFactory.client_transport = xport
         ip,ports = addr.split(':')
         c.HubFactory.client_ip = ip
-        c.Global.sshserver = cfg['ssh']
+        self.ssh_server = cfg['ssh']
         assert int(ports) == c.HubFactory.regport, "regport mismatch"
     
-    def construct(self):
-        # This is the working dir by now.
-        sys.path.insert(0, '')
-        c = self.master_config
+    def init_hub(self):
+        c = self.config
         
-        self.import_statements()
-        reusing = c.Global.reuse_files
+        self.do_import_statements()
+        reusing = self.reuse_files
         if reusing:
             try:
                 self.load_config_from_json()
@@ -346,21 +239,20 b' class IPControllerApp(ApplicationWithClusterDir):'
         # check again, because reusing may have failed:
         if reusing:
             pass
-        elif c.Global.secure:
-            keyfile = os.path.join(c.Global.security_dir, c.Global.exec_key)
+        elif self.secure:
             key = str(uuid.uuid4())
-            with open(keyfile, 'w') as f:
-                f.write(key)
-            os.chmod(keyfile, stat.S_IRUSR|stat.S_IWUSR)
-            c.SessionFactory.exec_key = key
+            # keyfile = os.path.join(self.profile_dir.security_dir, self.exec_key)
+            # with open(keyfile, 'w') as f:
+            #     f.write(key)
+            # os.chmod(keyfile, stat.S_IRUSR|stat.S_IWUSR)
+            c.Session.key = key
         else:
-            c.SessionFactory.exec_key = ''
-            key = None
+            key = c.Session.key = ''
         
         try:
-            self.factory = ControllerFactory(config=c, logname=self.log.name)
-            self.start_logging()
-            self.factory.construct()
+            self.factory = HubFactory(config=c, log=self.log)
+            # self.start_logging()
+            self.factory.init_hub()
         except:
             self.log.error("Couldn't construct the Controller", exc_info=True)
             self.exit(1)
@@ -369,21 +261,82 b' class IPControllerApp(ApplicationWithClusterDir):'
             # save to new json config files
             f = self.factory
             cdict = {'exec_key' : key,
-                    'ssh' : c.Global.sshserver,
+                    'ssh' : self.ssh_server,
                     'url' : "%s://%s:%s"%(f.client_transport, f.client_ip, f.regport),
-                    'location' : c.Global.location
+                    'location' : self.location
                     }
             self.save_connection_dict('ipcontroller-client.json', cdict)
             edict = cdict
             edict['url']="%s://%s:%s"%((f.client_transport, f.client_ip, f.regport))
             self.save_connection_dict('ipcontroller-engine.json', edict)
+
+    #
+    def init_schedulers(self):
+        children = self.children
+        mq = import_item(str(self.mq_class))
         
+        hub = self.factory
+        # maybe_inproc = 'inproc://monitor' if self.use_threads else self.monitor_url
+        # IOPub relay (in a Process)
+        q = mq(zmq.PUB, zmq.SUB, zmq.PUB, 'N/A','iopub')
+        q.bind_in(hub.client_info['iopub'])
+        q.bind_out(hub.engine_info['iopub'])
+        q.setsockopt_out(zmq.SUBSCRIBE, '')
+        q.connect_mon(hub.monitor_url)
+        q.daemon=True
+        children.append(q)
+
+        # Multiplexer Queue (in a Process)
+        q = mq(zmq.XREP, zmq.XREP, zmq.PUB, 'in', 'out')
+        q.bind_in(hub.client_info['mux'])
+        q.setsockopt_in(zmq.IDENTITY, 'mux')
+        q.bind_out(hub.engine_info['mux'])
+        q.connect_mon(hub.monitor_url)
+        q.daemon=True
+        children.append(q)
+
+        # Control Queue (in a Process)
+        q = mq(zmq.XREP, zmq.XREP, zmq.PUB, 'incontrol', 'outcontrol')
+        q.bind_in(hub.client_info['control'])
+        q.setsockopt_in(zmq.IDENTITY, 'control')
+        q.bind_out(hub.engine_info['control'])
+        q.connect_mon(hub.monitor_url)
+        q.daemon=True
+        children.append(q)
+        try:
+            scheme = self.config.TaskScheduler.scheme_name
+        except AttributeError:
+            scheme = TaskScheduler.scheme_name.get_default_value()
+        # Task Queue (in a Process)
+        if scheme == 'pure':
+            self.log.warn("task::using pure XREQ Task scheduler")
+            q = mq(zmq.XREP, zmq.XREQ, zmq.PUB, 'intask', 'outtask')
+            # q.setsockopt_out(zmq.HWM, hub.hwm)
+            q.bind_in(hub.client_info['task'][1])
+            q.setsockopt_in(zmq.IDENTITY, 'task')
+            q.bind_out(hub.engine_info['task'])
+            q.connect_mon(hub.monitor_url)
+            q.daemon=True
+            children.append(q)
+        elif scheme == 'none':
+            self.log.warn("task::using no Task scheduler")
+
+        else:
+            self.log.info("task::using Python %s Task scheduler"%scheme)
+            sargs = (hub.client_info['task'][1], hub.engine_info['task'],
+                                hub.monitor_url, hub.client_info['notification'])
+            kwargs = dict(logname='scheduler', loglevel=self.log_level,
+                            log_url = self.log_url, config=dict(self.config))
+            q = Process(target=launch_scheduler, args=sargs, kwargs=kwargs)
+            q.daemon=True
+            children.append(q)
+
     
     def save_urls(self):
         """save the registration urls to files."""
-        c = self.master_config
+        c = self.config
         
-        sec_dir = c.Global.security_dir
+        sec_dir = self.profile_dir.security_dir
         cf = self.factory
         
         with open(os.path.join(sec_dir, 'ipcontroller-engine.url'), 'w') as f:
@@ -393,8 +346,8 b' class IPControllerApp(ApplicationWithClusterDir):'
             f.write("%s://%s:%s"%(cf.client_transport, cf.client_ip, cf.regport))
         
     
-    def import_statements(self):
-        statements = self.master_config.Global.import_statements
+    def do_import_statements(self):
+        statements = self.import_statements
         for s in statements:
             try:
                 self.log.msg("Executing statement: '%s'" % s)
@@ -402,30 +355,52 b' class IPControllerApp(ApplicationWithClusterDir):'
             except:
                 self.log.msg("Error running statement: %s" % s)
 
-    def start_logging(self):
-        super(IPControllerApp, self).start_logging()
-        if self.master_config.Global.log_url:
-            context = self.factory.context
+    def forward_logging(self):
+        if self.log_url:
+            self.log.info("Forwarding logging to %s"%self.log_url)
+            context = zmq.Context.instance()
             lsock = context.socket(zmq.PUB)
-            lsock.connect(self.master_config.Global.log_url)
+            lsock.connect(self.log_url)
             handler = PUBHandler(lsock)
+            self.log.removeHandler(self._log_handler)
             handler.root_topic = 'controller'
             handler.setLevel(self.log_level)
             self.log.addHandler(handler)
-    # 
-    def start_app(self):
+            self._log_handler = handler
+    # #
+    
+    def initialize(self, argv=None):
+        super(IPControllerApp, self).initialize(argv)
+        self.forward_logging()
+        self.init_hub()
+        self.init_schedulers()
+    
+    def start(self):
         # Start the subprocesses:
         self.factory.start()
+        child_procs = []
+        for child in self.children:
+            child.start()
+            if isinstance(child, ProcessMonitoredQueue):
+                child_procs.append(child.launcher)
+            elif isinstance(child, Process):
+                child_procs.append(child)
+        if child_procs:
+            signal_children(child_procs)
+
         self.write_pid_file(overwrite=True)
+
         try:
             self.factory.loop.start()
         except KeyboardInterrupt:
             self.log.critical("Interrupted, Exiting...\n")
+            
 
 
 def launch_new_instance():
     """Create and run the IPython controller"""
-    app = IPControllerApp()
+    app = IPControllerApp.instance()
+    app.initialize()
     app.start()
 
 

IPython/parallel/apps/ipengineapp.py

@@ -2,10 +2,16 b''
 # encoding: utf-8
 """
 The IPython engine application
+
+Authors:
+
+* Brian Granger
+* MinRK
+
 """
 
 #-----------------------------------------------------------------------------
-#  Copyright (C) 2008-2009  The IPython Development Team
+#  Copyright (C) 2008-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
@@ -22,18 +28,18 b' import sys'
 import zmq
 from zmq.eventloop import ioloop
 
-from IPython.parallel.apps.clusterdir import (
-    ApplicationWithClusterDir,
-    ClusterDirConfigLoader
-)
+from IPython.core.profiledir import ProfileDir
+from IPython.parallel.apps.baseapp import BaseParallelApplication
 from IPython.zmq.log import EnginePUBHandler
 
-from IPython.parallel import factory
+from IPython.config.configurable import Configurable
+from IPython.zmq.session import Session
 from IPython.parallel.engine.engine import EngineFactory
 from IPython.parallel.engine.streamkernel import Kernel
 from IPython.parallel.util import disambiguate_url
 
 from IPython.utils.importstring import import_item
+from IPython.utils.traitlets import Bool, Unicode, Dict, List
 
 
 #-----------------------------------------------------------------------------
@@ -43,6 +49,20 b' from IPython.utils.importstring import import_item'
 #: The default config file name for this application
 default_config_file_name = u'ipengine_config.py'
 
+_description = """Start an IPython engine for parallel computing.
+
+IPython engines run in parallel and perform computations on behalf of a client
+and controller. A controller needs to be started before the engines. The
+engine can be configured using command line options or using a cluster
+directory. Cluster directories contain config, log and security files and are
+usually located in your ipython directory and named as "profile_name".
+See the `profile` and `profile_dir` options for details.
+"""
+
+
+#-----------------------------------------------------------------------------
+# MPI configuration
+#-----------------------------------------------------------------------------
 
 mpi4py_init = """from mpi4py import MPI as mpi
 mpi.size = mpi.COMM_WORLD.Get_size()
@@ -58,123 +78,78 b' mpi.rank = 0'
 mpi.size = 0
 """
 
+class MPI(Configurable):
+    """Configurable for MPI initialization"""
+    use = Unicode('', config=True,
+        help='How to enable MPI (mpi4py, pytrilinos, or empty string to disable).'
+        )
 
-_description = """Start an IPython engine for parallel computing.\n\n
+    def _on_use_changed(self, old, new):
+        # load default init script if it's not set
+        if not self.init_script:
+            self.init_script = self.default_inits.get(new, '')
+
+    init_script = Unicode('', config=True,
+        help="Initialization code for MPI")
+
+    default_inits = Dict({'mpi4py' : mpi4py_init, 'pytrilinos':pytrilinos_init},
+        config=True)
 
-IPython engines run in parallel and perform computations on behalf of a client
-and controller. A controller needs to be started before the engines. The
-engine can be configured using command line options or using a cluster
-directory. Cluster directories contain config, log and security files and are
-usually located in your ipython directory and named as "cluster_<profile>".
-See the --profile and --cluster-dir options for details.
-"""
 
 #-----------------------------------------------------------------------------
-# Command line options
+# Main application
 #-----------------------------------------------------------------------------
 
 
-class IPEngineAppConfigLoader(ClusterDirConfigLoader):
-
-    def _add_arguments(self):
-        super(IPEngineAppConfigLoader, self)._add_arguments()
-        paa = self.parser.add_argument
-        # Controller config
-        paa('--file', '-f',
-            type=unicode, dest='Global.url_file',
-            help='The full location of the file containing the connection information fo '
-            'controller. If this is not given, the file must be in the '
-            'security directory of the cluster directory.  This location is '
-            'resolved using the --profile and --app-dir options.',
-            metavar='Global.url_file')
-        # MPI
-        paa('--mpi',
-            type=str, dest='MPI.use',
-            help='How to enable MPI (mpi4py, pytrilinos, or empty string to disable).',
-            metavar='MPI.use')
-        # Global config
-        paa('--log-to-file',
-            action='store_true', dest='Global.log_to_file',
-            help='Log to a file in the log directory (default is stdout)')
-        paa('--log-url',
-            dest='Global.log_url',
-            help="url of ZMQ logger, as started with iploggerz")
-        # paa('--execkey',
-        #     type=str, dest='Global.exec_key',
-        #     help='path to a file containing an execution key.',
-        #     metavar='keyfile')
-        # paa('--no-secure',
-        #     action='store_false', dest='Global.secure',
-        #     help='Turn off execution keys.')
-        # paa('--secure',
-        #     action='store_true', dest='Global.secure',
-        #     help='Turn on execution keys (default).')
-        # init command
-        paa('-c',
-            type=str, dest='Global.extra_exec_lines',
+class IPEngineApp(BaseParallelApplication):
+
+    name = Unicode(u'ipengine')
+    description = Unicode(_description)
+    config_file_name = Unicode(default_config_file_name)
+    classes = List([ProfileDir, Session, EngineFactory, Kernel, MPI])
+
+    startup_script = Unicode(u'', config=True,
+        help='specify a script to be run at startup')
+    startup_command = Unicode('', config=True,
             help='specify a command to be run at startup')
-        paa('-s',
-            type=unicode, dest='Global.extra_exec_file',
-            help='specify a script to be run at startup')
-        
-        factory.add_session_arguments(self.parser)
-        factory.add_registration_arguments(self.parser)
 
+    url_file = Unicode(u'', config=True,
+        help="""The full location of the file containing the connection information for
+        the controller. If this is not given, the file must be in the
+        security directory of the cluster directory.  This location is
+        resolved using the `profile` or `profile_dir` options.""",
+        )
 
-#-----------------------------------------------------------------------------
-# Main application
-#-----------------------------------------------------------------------------
+    url_file_name = Unicode(u'ipcontroller-engine.json')
+    log_url = Unicode('', config=True,
+        help="""The URL for the iploggerapp instance, for forwarding
+        logging to a central location.""")
 
+    aliases = Dict(dict(
+        file = 'IPEngineApp.url_file',
+        c = 'IPEngineApp.startup_command',
+        s = 'IPEngineApp.startup_script',
 
-class IPEngineApp(ApplicationWithClusterDir):
-
-    name = u'ipengine'
-    description = _description
-    command_line_loader = IPEngineAppConfigLoader
-    default_config_file_name = default_config_file_name
-    auto_create_cluster_dir = True
-
-    def create_default_config(self):
-        super(IPEngineApp, self).create_default_config()
-
-        # The engine should not clean logs as we don't want to remove the
-        # active log files of other running engines.
-        self.default_config.Global.clean_logs = False
-        self.default_config.Global.secure = True
-
-        # Global config attributes
-        self.default_config.Global.exec_lines = []
-        self.default_config.Global.extra_exec_lines = ''
-        self.default_config.Global.extra_exec_file = u''
-
-        # Configuration related to the controller
-        # This must match the filename (path not included) that the controller
-        # used for the FURL file.
-        self.default_config.Global.url_file = u''
-        self.default_config.Global.url_file_name = u'ipcontroller-engine.json'
-        # If given, this is the actual location of the controller's FURL file.
-        # If not, this is computed using the profile, app_dir and furl_file_name
-        # self.default_config.Global.key_file_name = u'exec_key.key'
-        # self.default_config.Global.key_file = u''
-
-        # MPI related config attributes
-        self.default_config.MPI.use = ''
-        self.default_config.MPI.mpi4py = mpi4py_init
-        self.default_config.MPI.pytrilinos = pytrilinos_init
-
-    def post_load_command_line_config(self):
-        pass
-
-    def pre_construct(self):
-        super(IPEngineApp, self).pre_construct()
-        # self.find_cont_url_file()
-        self.find_url_file()
-        if self.master_config.Global.extra_exec_lines:
-            self.master_config.Global.exec_lines.append(self.master_config.Global.extra_exec_lines)
-        if self.master_config.Global.extra_exec_file:
-            enc = sys.getfilesystemencoding() or 'utf8'
-            cmd="execfile(%r)"%self.master_config.Global.extra_exec_file.encode(enc)
-            self.master_config.Global.exec_lines.append(cmd)
+        ident = 'Session.session',
+        user = 'Session.username',
+        exec_key = 'Session.keyfile',
+
+        url = 'EngineFactory.url',
+        ip = 'EngineFactory.ip',
+        transport = 'EngineFactory.transport',
+        port = 'EngineFactory.regport',
+        location = 'EngineFactory.location',
+
+        timeout = 'EngineFactory.timeout',
+
+        profile = "IPEngineApp.profile",
+        profile_dir = 'ProfileDir.location',
+
+        mpi = 'MPI.use',
+
+        log_level = 'IPEngineApp.log_level',
+        log_url = 'IPEngineApp.log_url'
+    ))
 
     # def find_key_file(self):
     #     """Set the key file.
@@ -186,7 +161,7 b' class IPEngineApp(ApplicationWithClusterDir):'
     #     # Find the actual controller key file
     #     if not config.Global.key_file:
     #         try_this = os.path.join(
-    #             config.Global.cluster_dir, 
+    #             config.Global.profile_dir, 
     #             config.Global.security_dir,
     #             config.Global.key_file_name
     #         )
@@ -198,82 +173,74 b' class IPEngineApp(ApplicationWithClusterDir):'
         Here we don't try to actually see if it exists for is valid as that
         is hadled by the connection logic.
         """
-        config = self.master_config
+        config = self.config
         # Find the actual controller key file
-        if not config.Global.url_file:
-            try_this = os.path.join(
-                config.Global.cluster_dir, 
-                config.Global.security_dir,
-                config.Global.url_file_name
+        if not self.url_file:
+            self.url_file = os.path.join(
+                self.profile_dir.security_dir,
+                self.url_file_name
             )
-            config.Global.url_file = try_this
-        
-    def construct(self):
+    def init_engine(self):
         # This is the working dir by now.
         sys.path.insert(0, '')
-        config = self.master_config
+        config = self.config
+        # print config
+        self.find_url_file()
+
         # if os.path.exists(config.Global.key_file) and config.Global.secure:
         #     config.SessionFactory.exec_key = config.Global.key_file
-        if os.path.exists(config.Global.url_file):
-            with open(config.Global.url_file) as f:
+        if os.path.exists(self.url_file):
+            with open(self.url_file) as f:
                 d = json.loads(f.read())
                 for k,v in d.iteritems():
                     if isinstance(v, unicode):
                         d[k] = v.encode()
             if d['exec_key']:
-                config.SessionFactory.exec_key = d['exec_key']
+                config.Session.key = d['exec_key']
             d['url'] = disambiguate_url(d['url'], d['location'])
-            config.RegistrationFactory.url=d['url']
+            config.EngineFactory.url = d['url']
             config.EngineFactory.location = d['location']
         
+        try:
+            exec_lines = config.Kernel.exec_lines
+        except AttributeError:
+            config.Kernel.exec_lines = []
+            exec_lines = config.Kernel.exec_lines
         
-        
-        config.Kernel.exec_lines = config.Global.exec_lines
-
-        self.start_mpi()
+        if self.startup_script:
+            enc = sys.getfilesystemencoding() or 'utf8'
+            cmd="execfile(%r)"%self.startup_script.encode(enc)
+            exec_lines.append(cmd)
+        if self.startup_command:
+            exec_lines.append(self.startup_command)
 
-        # Create the underlying shell class and EngineService
+        # Create the underlying shell class and Engine
         # shell_class = import_item(self.master_config.Global.shell_class)
+        # print self.config
         try:
-            self.engine = EngineFactory(config=config, logname=self.log.name)
+            self.engine = EngineFactory(config=config, log=self.log)
         except:
             self.log.error("Couldn't start the Engine", exc_info=True)
             self.exit(1)
         
-        self.start_logging()
-
-        # Create the service hierarchy
-        # self.main_service = service.MultiService()
-        # self.engine_service.setServiceParent(self.main_service)
-        # self.tub_service = Tub()
-        # self.tub_service.setServiceParent(self.main_service)
-        # # This needs to be called before the connection is initiated
-        # self.main_service.startService()
-
-        # This initiates the connection to the controller and calls
-        # register_engine to tell the controller we are ready to do work
-        # self.engine_connector = EngineConnector(self.tub_service)
-
-        # self.log.info("Using furl file: %s" % self.master_config.Global.furl_file)
-
-        # reactor.callWhenRunning(self.call_connect)
-
-
-    def start_logging(self):
-        super(IPEngineApp, self).start_logging()
-        if self.master_config.Global.log_url:
+    def forward_logging(self):
+        if self.log_url:
+            self.log.info("Forwarding logging to %s"%self.log_url)
             context = self.engine.context
             lsock = context.socket(zmq.PUB)
-            lsock.connect(self.master_config.Global.log_url)
+            lsock.connect(self.log_url)
+            self.log.removeHandler(self._log_handler)
             handler = EnginePUBHandler(self.engine, lsock)
             handler.setLevel(self.log_level)
             self.log.addHandler(handler)
-    
-    def start_mpi(self):
+            self._log_handler = handler
+    #
+    def init_mpi(self):
         global mpi
-        mpikey = self.master_config.MPI.use
-        mpi_import_statement = self.master_config.MPI.get(mpikey, None)
-        if mpi_import_statement is not None:
+        self.mpi = MPI(config=self.config)
+
+        mpi_import_statement = self.mpi.init_script
+        if mpi_import_statement:
             try:
                 self.log.info("Initializing MPI:")
                 self.log.info(mpi_import_statement)
@@ -283,8 +250,13 b' class IPEngineApp(ApplicationWithClusterDir):'
         else:
             mpi = None
 
-
-    def start_app(self):
+    def initialize(self, argv=None):
+        super(IPEngineApp, self).initialize(argv)
+        self.init_mpi()
+        self.init_engine()
+        self.forward_logging()
+    
+    def start(self):
         self.engine.start()
         try:
             self.engine.loop.start()
@@ -293,8 +265,9 b' class IPEngineApp(ApplicationWithClusterDir):'
 
 
 def launch_new_instance():
-    """Create and run the IPython controller"""
-    app = IPEngineApp()
+    """Create and run the IPython engine"""
+    app = IPEngineApp.instance()
+    app.initialize()
     app.start()
 
 

IPython/parallel/apps/iploggerapp.py

@@ -2,6 +2,11 b''
 # encoding: utf-8
 """
 A simple IPython logger application
+
+Authors:
+
+* MinRK
+
 """
 
 #-----------------------------------------------------------------------------
@@ -20,9 +25,12 b' import sys'
 
 import zmq
 
-from IPython.parallel.apps.clusterdir import (
-    ApplicationWithClusterDir,
-    ClusterDirConfigLoader
+from IPython.core.profiledir import ProfileDir
+from IPython.utils.traitlets import Bool, Dict, Unicode
+
+from IPython.parallel.apps.baseapp import (
+    BaseParallelApplication,
+    base_aliases
 )
 from IPython.parallel.apps.logwatcher import LogWatcher
 
@@ -33,89 +41,49 b' from IPython.parallel.apps.logwatcher import LogWatcher'
 #: The default config file name for this application
 default_config_file_name = u'iplogger_config.py'
 
-_description = """Start an IPython logger for parallel computing.\n\n
+_description = """Start an IPython logger for parallel computing.
 
 IPython controllers and engines (and your own processes) can broadcast log messages
 by registering a `zmq.log.handlers.PUBHandler` with the `logging` module. The
 logger can be configured using command line options or using a cluster
 directory. Cluster directories contain config, log and security files and are
-usually located in your ipython directory and named as "cluster_<profile>".
-See the --profile and --cluster-dir options for details.
+usually located in your ipython directory and named as "profile_name".
+See the `profile` and `profile_dir` options for details.
 """
 
-#-----------------------------------------------------------------------------
-# Command line options
-#-----------------------------------------------------------------------------
-
-
-class IPLoggerAppConfigLoader(ClusterDirConfigLoader):
-
-    def _add_arguments(self):
-        super(IPLoggerAppConfigLoader, self)._add_arguments()
-        paa = self.parser.add_argument
-        # Controller config
-        paa('--url',
-            type=str, dest='LogWatcher.url',
-            help='The url the LogWatcher will listen on',
-            )
-        # MPI
-        paa('--topics',
-            type=str, dest='LogWatcher.topics', nargs='+',
-            help='What topics to subscribe to',
-            metavar='topics')
-        # Global config
-        paa('--log-to-file',
-            action='store_true', dest='Global.log_to_file',
-            help='Log to a file in the log directory (default is stdout)')
-        
 
 #-----------------------------------------------------------------------------
 # Main application
 #-----------------------------------------------------------------------------
+aliases = {}
+aliases.update(base_aliases)
+aliases.update(dict(url='LogWatcher.url', topics='LogWatcher.topics'))
 
+class IPLoggerApp(BaseParallelApplication):
 
-class IPLoggerApp(ApplicationWithClusterDir):
-
-    name = u'iploggerz'
+    name = u'iplogger'
     description = _description
-    command_line_loader = IPLoggerAppConfigLoader
-    default_config_file_name = default_config_file_name
-    auto_create_cluster_dir = True
-
-    def create_default_config(self):
-        super(IPLoggerApp, self).create_default_config()
-
-        # The engine should not clean logs as we don't want to remove the
-        # active log files of other running engines.
-        self.default_config.Global.clean_logs = False
-
-        # If given, this is the actual location of the logger's URL file.
-        # If not, this is computed using the profile, app_dir and furl_file_name
-        self.default_config.Global.url_file_name = u'iplogger.url'
-        self.default_config.Global.url_file = u''
-
-    def post_load_command_line_config(self):
-        pass
-
-    def pre_construct(self):
-        super(IPLoggerApp, self).pre_construct()
-
-    def construct(self):
-        # This is the working dir by now.
-        sys.path.insert(0, '')
-
-        self.start_logging()
-
+    config_file_name = Unicode(default_config_file_name)
+    
+    classes = [LogWatcher, ProfileDir]
+    aliases = Dict(aliases)
+
+    def initialize(self, argv=None):
+        super(IPLoggerApp, self).initialize(argv)
+        self.init_watcher()
+    
+    def init_watcher(self):
         try:
-            self.watcher = LogWatcher(config=self.master_config, logname=self.log.name)
+            self.watcher = LogWatcher(config=self.config, log=self.log)
         except:
             self.log.error("Couldn't start the LogWatcher", exc_info=True)
             self.exit(1)
+        self.log.info("Listening for log messages on %r"%self.watcher.url)
         
 
-    def start_app(self):
+    def start(self):
+        self.watcher.start()
         try:
-            self.watcher.start()
             self.watcher.loop.start()
         except KeyboardInterrupt:
             self.log.critical("Logging Interrupted, shutting down...\n")
@@ -123,7 +91,8 b' class IPLoggerApp(ApplicationWithClusterDir):'
 
 def launch_new_instance():
     """Create and run the IPython LogWatcher"""
-    app = IPLoggerApp()
+    app = IPLoggerApp.instance()
+    app.initialize()
     app.start()
 
 

IPython/parallel/apps/launcher.py

@@ -2,10 +2,15 b''
 # encoding: utf-8
 """
 Facilities for launching IPython processes asynchronously.
+
+Authors:
+
+* Brian Granger
+* MinRK
 """
 
 #-----------------------------------------------------------------------------
-#  Copyright (C) 2008-2009  The IPython Development Team
+#  Copyright (C) 2008-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
@@ -49,14 +54,13 b' except ImportError:'
 
 from zmq.eventloop import ioloop
 
-from IPython.external import Itpl
-# from IPython.config.configurable import Configurable
-from IPython.utils.traitlets import Any, Str, Int, List, Unicode, Dict, Instance, CUnicode
+from IPython.config.application import Application
+from IPython.config.configurable import LoggingConfigurable
+from IPython.utils.text import EvalFormatter
+from IPython.utils.traitlets import Any, Int, List, Unicode, Dict, Instance
 from IPython.utils.path import get_ipython_module_path
 from IPython.utils.process import find_cmd, pycmd2argv, FindCmdError
 
-from IPython.parallel.factory import LoggingFactory
-
 from .win32support import forward_read_events
 
 from .winhpcjob import IPControllerTask, IPEngineTask, IPControllerJob, IPEngineSetJob
@@ -97,16 +101,16 b' class UnknownStatus(LauncherError):'
     pass
 
 
-class BaseLauncher(LoggingFactory):
+class BaseLauncher(LoggingConfigurable):
     """An asbtraction for starting, stopping and signaling a process."""
 
     # In all of the launchers, the work_dir is where child processes will be
-    # run. This will usually be the cluster_dir, but may not be. any work_dir
+    # run. This will usually be the profile_dir, but may not be. any work_dir
     # passed into the __init__ method will override the config value.
     # This should not be used to set the work_dir for the actual engine
     # and controller. Instead, use their own config files or the
     # controller_args, engine_args attributes of the launchers to add
-    # the --work-dir option.
+    # the work_dir option.
     work_dir = Unicode(u'.')
     loop = Instance('zmq.eventloop.ioloop.IOLoop')
     
@@ -153,29 +157,20 b' class BaseLauncher(LoggingFactory):'
             return False
 
     def start(self):
-        """Start the process.
-
-        This must return a deferred that fires with information about the
-        process starting (like a pid, job id, etc.).
-        """
+        """Start the process."""
         raise NotImplementedError('start must be implemented in a subclass')
 
     def stop(self):
         """Stop the process and notify observers of stopping.
 
-        This must return a deferred that fires with information about the
-        processing stopping, like errors that occur while the process is
-        attempting to be shut down. This deferred won't fire when the process
-        actually stops. To observe the actual process stopping, see
-        :func:`observe_stop`.
+        This method will return None immediately.
+        To observe the actual process stopping, see :meth:`on_stop`.
         """
         raise NotImplementedError('stop must be implemented in a subclass')
 
     def on_stop(self, f):
-        """Get a deferred that will fire when the process stops.
-
-        The deferred will fire with data that contains information about
-        the exit status of the process.
+        """Register a callback to be called with this Launcher's stop_data
+        when the process actually finishes.
         """
         if self.state=='after':
             return f(self.stop_data)
@@ -198,7 +193,7 b' class BaseLauncher(LoggingFactory):'
         """Call this to trigger process stop actions.
 
         This logs the process stopping and sets the state to 'after'. Call
-        this to trigger all the deferreds from :func:`observe_stop`."""
+        this to trigger callbacks registered via :meth:`on_stop`."""
 
         self.log.info('Process %r stopped: %r' % (self.args[0], data))
         self.stop_data = data
@@ -211,8 +206,6 b' class BaseLauncher(LoggingFactory):'
     def signal(self, sig):
         """Signal the process.
 
-        Return a semi-meaningless deferred after signaling the process.
-
         Parameters
         ----------
         sig : str or int
@@ -243,7 +236,6 b' class LocalProcessLauncher(BaseLauncher):'
             work_dir=work_dir, config=config, **kwargs
         )
         self.process = None
-        self.start_deferred = None
         self.poller = None
 
     def find_args(self):
@@ -328,17 +320,19 b' class LocalProcessLauncher(BaseLauncher):'
 class LocalControllerLauncher(LocalProcessLauncher):
     """Launch a controller as a regular external process."""
 
-    controller_cmd = List(ipcontroller_cmd_argv, config=True)
+    controller_cmd = List(ipcontroller_cmd_argv, config=True,
+        help="""Popen command to launch ipcontroller.""")
     # Command line arguments to ipcontroller.
-    controller_args = List(['--log-to-file','--log-level', str(logging.INFO)], config=True)
+    controller_args = List(['--log-to-file','log_level=%i'%logging.INFO], config=True,
+        help="""command-line args to pass to ipcontroller""")
 
     def find_args(self):
         return self.controller_cmd + self.controller_args
 
-    def start(self, cluster_dir):
-        """Start the controller by cluster_dir."""
-        self.controller_args.extend(['--cluster-dir', cluster_dir])
-        self.cluster_dir = unicode(cluster_dir)
+    def start(self, profile_dir):
+        """Start the controller by profile_dir."""
+        self.controller_args.extend(['profile_dir=%s'%profile_dir])
+        self.profile_dir = unicode(profile_dir)
         self.log.info("Starting LocalControllerLauncher: %r" % self.args)
         return super(LocalControllerLauncher, self).start()
 
@@ -346,19 +340,20 b' class LocalControllerLauncher(LocalProcessLauncher):'
 class LocalEngineLauncher(LocalProcessLauncher):
     """Launch a single engine as a regular externall process."""
 
-    engine_cmd = List(ipengine_cmd_argv, config=True)
+    engine_cmd = List(ipengine_cmd_argv, config=True,
+        help="""command to launch the Engine.""")
     # Command line arguments for ipengine.
-    engine_args = List(
-        ['--log-to-file','--log-level', str(logging.INFO)], config=True
+    engine_args = List(['--log-to-file','log_level=%i'%logging.INFO], config=True,
+        help="command-line arguments to pass to ipengine"
     )
 
     def find_args(self):
         return self.engine_cmd + self.engine_args
 
-    def start(self, cluster_dir):
-        """Start the engine by cluster_dir."""
-        self.engine_args.extend(['--cluster-dir', cluster_dir])
-        self.cluster_dir = unicode(cluster_dir)
+    def start(self, profile_dir):
+        """Start the engine by profile_dir."""
+        self.engine_args.extend(['profile_dir=%s'%profile_dir])
+        self.profile_dir = unicode(profile_dir)
         return super(LocalEngineLauncher, self).start()
 
 
@@ -367,7 +362,8 b' class LocalEngineSetLauncher(BaseLauncher):'
 
     # Command line arguments for ipengine.
     engine_args = List(
-        ['--log-to-file','--log-level', str(logging.INFO)], config=True
+        ['--log-to-file','log_level=%i'%logging.INFO], config=True,
+        help="command-line arguments to pass to ipengine"
     )
     # launcher class
     launcher_class = LocalEngineLauncher
@@ -381,16 +377,16 b' class LocalEngineSetLauncher(BaseLauncher):'
         )
         self.stop_data = {}
 
-    def start(self, n, cluster_dir):
-        """Start n engines by profile or cluster_dir."""
-        self.cluster_dir = unicode(cluster_dir)
+    def start(self, n, profile_dir):
+        """Start n engines by profile or profile_dir."""
+        self.profile_dir = unicode(profile_dir)
         dlist = []
         for i in range(n):
-            el = self.launcher_class(work_dir=self.work_dir, config=self.config, logname=self.log.name)
+            el = self.launcher_class(work_dir=self.work_dir, config=self.config, log=self.log)
             # Copy the engine args over to each engine launcher.
             el.engine_args = copy.deepcopy(self.engine_args)
             el.on_stop(self._notice_engine_stopped)
-            d = el.start(cluster_dir)
+            d = el.start(profile_dir)
             if i==0:
                 self.log.info("Starting LocalEngineSetLauncher: %r" % el.args)
             self.launchers[i] = el
@@ -442,16 +438,18 b' class LocalEngineSetLauncher(BaseLauncher):'
 class MPIExecLauncher(LocalProcessLauncher):
     """Launch an external process using mpiexec."""
 
-    # The mpiexec command to use in starting the process.
-    mpi_cmd = List(['mpiexec'], config=True)
-    # The command line arguments to pass to mpiexec.
-    mpi_args = List([], config=True)
-    # The program to start using mpiexec.
-    program = List(['date'], config=True)
-    # The command line argument to the program.
-    program_args = List([], config=True)
-    # The number of instances of the program to start.
-    n = Int(1, config=True)
+    mpi_cmd = List(['mpiexec'], config=True,
+        help="The mpiexec command to use in starting the process."
+    )
+    mpi_args = List([], config=True,
+        help="The command line arguments to pass to mpiexec."
+    )
+    program = List(['date'], config=True,
+        help="The program to start via mpiexec.")
+    program_args = List([], config=True,
+        help="The command line argument to the program."
+    )
+    n = Int(1)
 
     def find_args(self):
         """Build self.args using all the fields."""
@@ -467,15 +465,18 b' class MPIExecLauncher(LocalProcessLauncher):'
 class MPIExecControllerLauncher(MPIExecLauncher):
     """Launch a controller using mpiexec."""
 
-    controller_cmd = List(ipcontroller_cmd_argv, config=True)
-    # Command line arguments to ipcontroller.
-    controller_args = List(['--log-to-file','--log-level', str(logging.INFO)], config=True)
-    n = Int(1, config=False)
+    controller_cmd = List(ipcontroller_cmd_argv, config=True,
+        help="Popen command to launch the Contropper"
+    )
+    controller_args = List(['--log-to-file','log_level=%i'%logging.INFO], config=True,
+        help="Command line arguments to pass to ipcontroller."
+    )
+    n = Int(1)
 
-    def start(self, cluster_dir):
-        """Start the controller by cluster_dir."""
-        self.controller_args.extend(['--cluster-dir', cluster_dir])
-        self.cluster_dir = unicode(cluster_dir)
+    def start(self, profile_dir):
+        """Start the controller by profile_dir."""
+        self.controller_args.extend(['profile_dir=%s'%profile_dir])
+        self.profile_dir = unicode(profile_dir)
         self.log.info("Starting MPIExecControllerLauncher: %r" % self.args)
         return super(MPIExecControllerLauncher, self).start(1)
 
@@ -486,17 +487,19 b' class MPIExecControllerLauncher(MPIExecLauncher):'
 
 class MPIExecEngineSetLauncher(MPIExecLauncher):
 
-    program = List(ipengine_cmd_argv, config=True)
-    # Command line arguments for ipengine.
+    program = List(ipengine_cmd_argv, config=True,
+        help="Popen command for ipengine"
+    )
     program_args = List(
-        ['--log-to-file','--log-level', str(logging.INFO)], config=True
+        ['--log-to-file','log_level=%i'%logging.INFO], config=True,
+        help="Command line arguments for ipengine."
     )
-    n = Int(1, config=True)
+    n = Int(1)
 
-    def start(self, n, cluster_dir):
-        """Start n engines by profile or cluster_dir."""
-        self.program_args.extend(['--cluster-dir', cluster_dir])
-        self.cluster_dir = unicode(cluster_dir)
+    def start(self, n, profile_dir):
+        """Start n engines by profile or profile_dir."""
+        self.program_args.extend(['profile_dir=%s'%profile_dir])
+        self.profile_dir = unicode(profile_dir)
         self.n = n
         self.log.info('Starting MPIExecEngineSetLauncher: %r' % self.args)
         return super(MPIExecEngineSetLauncher, self).start(n)
@@ -505,7 +508,7 b' class MPIExecEngineSetLauncher(MPIExecLauncher):'
 # SSH launchers
 #-----------------------------------------------------------------------------
 
-# TODO: Get SSH Launcher working again.
+# TODO: Get SSH Launcher back to level of sshx in 0.10.2
 
 class SSHLauncher(LocalProcessLauncher):
     """A minimal launcher for ssh.
@@ -515,13 +518,20 b' class SSHLauncher(LocalProcessLauncher):'
     as well.
     """
 
-    ssh_cmd = List(['ssh'], config=True)
-    ssh_args = List(['-tt'], config=True)
-    program = List(['date'], config=True)
-    program_args = List([], config=True)
-    hostname = CUnicode('', config=True)
-    user = CUnicode('', config=True)
-    location = CUnicode('')
+    ssh_cmd = List(['ssh'], config=True,
+        help="command for starting ssh")
+    ssh_args = List(['-tt'], config=True,
+        help="args to pass to ssh")
+    program = List(['date'], config=True,
+        help="Program to launch via ssh")
+    program_args = List([], config=True,
+        help="args to pass to remote program")
+    hostname = Unicode('', config=True,
+        help="hostname on which to launch the program")
+    user = Unicode('', config=True,
+        help="username for ssh")
+    location = Unicode('', config=True,
+        help="user@hostname location for ssh in one setting")
 
     def _hostname_changed(self, name, old, new):
         if self.user:
@@ -536,8 +546,8 b' class SSHLauncher(LocalProcessLauncher):'
         return self.ssh_cmd + self.ssh_args + [self.location] + \
                self.program + self.program_args
 
-    def start(self, cluster_dir, hostname=None, user=None):
-        self.cluster_dir = unicode(cluster_dir)
+    def start(self, profile_dir, hostname=None, user=None):
+        self.profile_dir = unicode(profile_dir)
         if hostname is not None:
             self.hostname = hostname
         if user is not None:
@@ -555,28 +565,33 b' class SSHLauncher(LocalProcessLauncher):'
 
 class SSHControllerLauncher(SSHLauncher):
 
-    program = List(ipcontroller_cmd_argv, config=True)
-    # Command line arguments to ipcontroller.
-    program_args = List(['-r', '--log-to-file','--log-level', str(logging.INFO)], config=True)
+    program = List(ipcontroller_cmd_argv, config=True,
+        help="remote ipcontroller command.")
+    program_args = List(['--reuse-files', '--log-to-file','log_level=%i'%logging.INFO], config=True,
+        help="Command line arguments to ipcontroller.")
 
 
 class SSHEngineLauncher(SSHLauncher):
-    program = List(ipengine_cmd_argv, config=True)
+    program = List(ipengine_cmd_argv, config=True,
+        help="remote ipengine command.")
     # Command line arguments for ipengine.
     program_args = List(
-        ['--log-to-file','--log-level', str(logging.INFO)], config=True
+        ['--log-to-file','log_level=%i'%logging.INFO], config=True,
+        help="Command line arguments to ipengine."
     )
     
 class SSHEngineSetLauncher(LocalEngineSetLauncher):
     launcher_class = SSHEngineLauncher
-    engines = Dict(config=True)
+    engines = Dict(config=True,
+        help="""dict of engines to launch.  This is a dict by hostname of ints,
+        corresponding to the number of engines to start on that host.""")
     
-    def start(self, n, cluster_dir):
-        """Start engines by profile or cluster_dir.
+    def start(self, n, profile_dir):
+        """Start engines by profile or profile_dir.
         `n` is ignored, and the `engines` config property is used instead.
         """
         
-        self.cluster_dir = unicode(cluster_dir)
+        self.profile_dir = unicode(profile_dir)
         dlist = []
         for host, n in self.engines.iteritems():
             if isinstance(n, (tuple, list)):
@@ -589,13 +604,13 b' class SSHEngineSetLauncher(LocalEngineSetLauncher):'
             else:
                 user=None
             for i in range(n):
-                el = self.launcher_class(work_dir=self.work_dir, config=self.config, logname=self.log.name)
+                el = self.launcher_class(work_dir=self.work_dir, config=self.config, log=self.log)
                 
                 # Copy the engine args over to each engine launcher.
                 i
                 el.program_args = args
                 el.on_stop(self._notice_engine_stopped)
-                d = el.start(cluster_dir, user=user, hostname=host)
+                d = el.start(profile_dir, user=user, hostname=host)
                 if i==0:
                     self.log.info("Starting SSHEngineSetLauncher: %r" % el.args)
                 self.launchers[host+str(i)] = el
@@ -624,17 +639,19 b' def find_job_cmd():'
 
 class WindowsHPCLauncher(BaseLauncher):
 
-    # A regular expression used to get the job id from the output of the 
-    # submit_command.
-    job_id_regexp = Str(r'\d+', config=True)
-    # The filename of the instantiated job script.
-    job_file_name = CUnicode(u'ipython_job.xml', config=True)
+    job_id_regexp = Unicode(r'\d+', config=True,
+        help="""A regular expression used to get the job id from the output of the
+        submit_command. """
+        )
+    job_file_name = Unicode(u'ipython_job.xml', config=True,
+        help="The filename of the instantiated job script.")
     # The full path to the instantiated job script. This gets made dynamically
     # by combining the work_dir with the job_file_name.
-    job_file = CUnicode(u'')
-    # The hostname of the scheduler to submit the job to
-    scheduler = CUnicode('', config=True)
-    job_cmd = CUnicode(find_job_cmd(), config=True)
+    job_file = Unicode(u'')
+    scheduler = Unicode('', config=True,
+        help="The hostname of the scheduler to submit the job to.")
+    job_cmd = Unicode(find_job_cmd(), config=True,
+        help="The command for submitting jobs.")
 
     def __init__(self, work_dir=u'.', config=None, **kwargs):
         super(WindowsHPCLauncher, self).__init__(
@@ -671,7 +688,7 b' class WindowsHPCLauncher(BaseLauncher):'
             '/scheduler:%s' % self.scheduler
         ]
         self.log.info("Starting Win HPC Job: %s" % (self.job_cmd + ' ' + ' '.join(args),))
-        # Twisted will raise DeprecationWarnings if we try to pass unicode to this
+
         output = check_output([self.job_cmd]+args,
             env=os.environ,
             cwd=self.work_dir,
@@ -702,8 +719,10 b' class WindowsHPCLauncher(BaseLauncher):'
 
 class WindowsHPCControllerLauncher(WindowsHPCLauncher):
 
-    job_file_name = CUnicode(u'ipcontroller_job.xml', config=True)
-    extra_args = List([], config=False)
+    job_file_name = Unicode(u'ipcontroller_job.xml', config=True,
+        help="WinHPC xml job file.")
+    extra_args = List([], config=False,
+        help="extra args to pass to ipcontroller")
 
     def write_job_file(self, n):
         job = IPControllerJob(config=self.config)
@@ -712,8 +731,8 b' class WindowsHPCControllerLauncher(WindowsHPCLauncher):'
         # The tasks work directory is *not* the actual work directory of 
         # the controller. It is used as the base path for the stdout/stderr
         # files that the scheduler redirects to.
-        t.work_directory = self.cluster_dir
-        # Add the --cluster-dir and from self.start().
+        t.work_directory = self.profile_dir
+        # Add the profile_dir and from self.start().
         t.controller_args.extend(self.extra_args)
         job.add_task(t)
 
@@ -722,19 +741,21 b' class WindowsHPCControllerLauncher(WindowsHPCLauncher):'
 
     @property
     def job_file(self):
-        return os.path.join(self.cluster_dir, self.job_file_name)
+        return os.path.join(self.profile_dir, self.job_file_name)
 
-    def start(self, cluster_dir):
-        """Start the controller by cluster_dir."""
-        self.extra_args = ['--cluster-dir', cluster_dir]
-        self.cluster_dir = unicode(cluster_dir)
+    def start(self, profile_dir):
+        """Start the controller by profile_dir."""
+        self.extra_args = ['profile_dir=%s'%profile_dir]
+        self.profile_dir = unicode(profile_dir)
         return super(WindowsHPCControllerLauncher, self).start(1)
 
 
 class WindowsHPCEngineSetLauncher(WindowsHPCLauncher):
 
-    job_file_name = CUnicode(u'ipengineset_job.xml', config=True)
-    extra_args = List([], config=False)
+    job_file_name = Unicode(u'ipengineset_job.xml', config=True,
+        help="jobfile for ipengines job")
+    extra_args = List([], config=False,
+        help="extra args to pas to ipengine")
 
     def write_job_file(self, n):
         job = IPEngineSetJob(config=self.config)
@@ -744,8 +765,8 b' class WindowsHPCEngineSetLauncher(WindowsHPCLauncher):'
             # The tasks work directory is *not* the actual work directory of 
             # the engine. It is used as the base path for the stdout/stderr
             # files that the scheduler redirects to.
-            t.work_directory = self.cluster_dir
-            # Add the --cluster-dir and from self.start().
+            t.work_directory = self.profile_dir
+            # Add the profile_dir and from self.start().
             t.engine_args.extend(self.extra_args)
             job.add_task(t)
 
@@ -754,12 +775,12 b' class WindowsHPCEngineSetLauncher(WindowsHPCLauncher):'
 
     @property
     def job_file(self):
-        return os.path.join(self.cluster_dir, self.job_file_name)
+        return os.path.join(self.profile_dir, self.job_file_name)
 
-    def start(self, n, cluster_dir):
-        """Start the controller by cluster_dir."""
-        self.extra_args = ['--cluster-dir', cluster_dir]
-        self.cluster_dir = unicode(cluster_dir)
+    def start(self, n, profile_dir):
+        """Start the controller by profile_dir."""
+        self.extra_args = ['profile_dir=%s'%profile_dir]
+        self.profile_dir = unicode(profile_dir)
         return super(WindowsHPCEngineSetLauncher, self).start(n)
 
 
@@ -776,41 +797,43 b' class BatchSystemLauncher(BaseLauncher):'
 
     This class also has the notion of a batch script. The ``batch_template``
     attribute can be set to a string that is a template for the batch script.
-    This template is instantiated using Itpl. Thus the template can use
-    ${n} fot the number of instances. Subclasses can add additional variables
+    This template is instantiated using string formatting. Thus the template can
+    use {n} fot the number of instances. Subclasses can add additional variables
     to the template dict.
     """
 
     # Subclasses must fill these in.  See PBSEngineSet
-    # The name of the command line program used to submit jobs.
-    submit_command = List([''], config=True)
-    # The name of the command line program used to delete jobs.
-    delete_command = List([''], config=True)
-    # A regular expression used to get the job id from the output of the 
-    # submit_command.
-    job_id_regexp = CUnicode('', config=True)
-    # The string that is the batch script template itself.
-    batch_template = CUnicode('', config=True)
-    # The file that contains the batch template
-    batch_template_file = CUnicode(u'', config=True)
-    # The filename of the instantiated batch script.
-    batch_file_name = CUnicode(u'batch_script', config=True)
-    # The PBS Queue
-    queue = CUnicode(u'', config=True)
+    submit_command = List([''], config=True,
+        help="The name of the command line program used to submit jobs.")
+    delete_command = List([''], config=True,
+        help="The name of the command line program used to delete jobs.")
+    job_id_regexp = Unicode('', config=True,
+        help="""A regular expression used to get the job id from the output of the
+        submit_command.""")
+    batch_template = Unicode('', config=True,
+        help="The string that is the batch script template itself.")
+    batch_template_file = Unicode(u'', config=True,
+        help="The file that contains the batch template.")
+    batch_file_name = Unicode(u'batch_script', config=True,
+        help="The filename of the instantiated batch script.")
+    queue = Unicode(u'', config=True,
+        help="The PBS Queue.")
     
     # not configurable, override in subclasses
     # PBS Job Array regex
-    job_array_regexp = CUnicode('')
-    job_array_template = CUnicode('')
+    job_array_regexp = Unicode('')
+    job_array_template = Unicode('')
     # PBS Queue regex
-    queue_regexp = CUnicode('')
-    queue_template = CUnicode('')
+    queue_regexp = Unicode('')
+    queue_template = Unicode('')
     # The default batch template, override in subclasses
-    default_template = CUnicode('')
+    default_template = Unicode('')
     # The full path to the instantiated batch script.
-    batch_file = CUnicode(u'')
+    batch_file = Unicode(u'')
     # the format dict used with batch_template:
     context = Dict()
+    # the Formatter instance for rendering the templates:
+    formatter = Instance(EvalFormatter, (), {})
 
     
     def find_args(self):
@@ -837,7 +860,6 b' class BatchSystemLauncher(BaseLauncher):'
         """Instantiate and write the batch script to the work_dir."""
         self.context['n'] = n
         self.context['queue'] = self.queue
-        print self.context
         # first priority is batch_template if set
         if self.batch_template_file and not self.batch_template:
             # second priority is batch_template_file
@@ -861,20 +883,19 b' class BatchSystemLauncher(BaseLauncher):'
             firstline, rest = self.batch_template.split('\n',1)
             self.batch_template = u'\n'.join([firstline, self.queue_template, rest])
             
-        script_as_string = Itpl.itplns(self.batch_template, self.context)
+        script_as_string = self.formatter.format(self.batch_template, **self.context)
         self.log.info('Writing instantiated batch script: %s' % self.batch_file)
     
         with open(self.batch_file, 'w') as f:
             f.write(script_as_string)
         os.chmod(self.batch_file, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
 
-    def start(self, n, cluster_dir):
+    def start(self, n, profile_dir):
         """Start n copies of the process using a batch system."""
-        # Here we save profile and cluster_dir in the context so they
-        # can be used in the batch script template as ${profile} and
-        # ${cluster_dir}
-        self.context['cluster_dir'] = cluster_dir
-        self.cluster_dir = unicode(cluster_dir)
+        # Here we save profile_dir in the context so they
+        # can be used in the batch script template as {profile_dir}
+        self.context['profile_dir'] = profile_dir
+        self.profile_dir = unicode(profile_dir)
         self.write_batch_script(n)
         output = check_output(self.args, env=os.environ)
         
@@ -891,84 +912,91 b' class BatchSystemLauncher(BaseLauncher):'
 class PBSLauncher(BatchSystemLauncher):
     """A BatchSystemLauncher subclass for PBS."""
 
-    submit_command = List(['qsub'], config=True)
-    delete_command = List(['qdel'], config=True)
-    job_id_regexp = CUnicode(r'\d+', config=True)
+    submit_command = List(['qsub'], config=True,
+        help="The PBS submit command ['qsub']")
+    delete_command = List(['qdel'], config=True,
+        help="The PBS delete command ['qsub']")
+    job_id_regexp = Unicode(r'\d+', config=True,
+        help="Regular expresion for identifying the job ID [r'\d+']")
     
-    batch_file = CUnicode(u'')
-    job_array_regexp = CUnicode('#PBS\W+-t\W+[\w\d\-\$]+')
-    job_array_template = CUnicode('#PBS -t 1-$n')
-    queue_regexp = CUnicode('#PBS\W+-q\W+\$?\w+')
-    queue_template = CUnicode('#PBS -q $queue')
+    batch_file = Unicode(u'')
+    job_array_regexp = Unicode('#PBS\W+-t\W+[\w\d\-\$]+')
+    job_array_template = Unicode('#PBS -t 1-{n}')
+    queue_regexp = Unicode('#PBS\W+-q\W+\$?\w+')
+    queue_template = Unicode('#PBS -q {queue}')
 
 
 class PBSControllerLauncher(PBSLauncher):
     """Launch a controller using PBS."""
 
-    batch_file_name = CUnicode(u'pbs_controller', config=True)
-    default_template= CUnicode("""#!/bin/sh
+    batch_file_name = Unicode(u'pbs_controller', config=True,
+        help="batch file name for the controller job.")
+    default_template= Unicode("""#!/bin/sh
 #PBS -V
 #PBS -N ipcontroller
-%s --log-to-file --cluster-dir $cluster_dir
+%s --log-to-file profile_dir={profile_dir}
 """%(' '.join(ipcontroller_cmd_argv)))
 
-    def start(self, cluster_dir):
-        """Start the controller by profile or cluster_dir."""
+    def start(self, profile_dir):
+        """Start the controller by profile or profile_dir."""
         self.log.info("Starting PBSControllerLauncher: %r" % self.args)
-        return super(PBSControllerLauncher, self).start(1, cluster_dir)
+        return super(PBSControllerLauncher, self).start(1, profile_dir)
 
 
 class PBSEngineSetLauncher(PBSLauncher):
     """Launch Engines using PBS"""
-    batch_file_name = CUnicode(u'pbs_engines', config=True)
-    default_template= CUnicode(u"""#!/bin/sh
+    batch_file_name = Unicode(u'pbs_engines', config=True,
+        help="batch file name for the engine(s) job.")
+    default_template= Unicode(u"""#!/bin/sh
 #PBS -V
 #PBS -N ipengine
-%s --cluster-dir $cluster_dir
+%s profile_dir={profile_dir}
 """%(' '.join(ipengine_cmd_argv)))
 
-    def start(self, n, cluster_dir):
-        """Start n engines by profile or cluster_dir."""
+    def start(self, n, profile_dir):
+        """Start n engines by profile or profile_dir."""
         self.log.info('Starting %i engines with PBSEngineSetLauncher: %r' % (n, self.args))
-        return super(PBSEngineSetLauncher, self).start(n, cluster_dir)
+        return super(PBSEngineSetLauncher, self).start(n, profile_dir)
 
 #SGE is very similar to PBS
 
 class SGELauncher(PBSLauncher):
     """Sun GridEngine is a PBS clone with slightly different syntax"""
-    job_array_regexp = CUnicode('#$$\W+-t\W+[\w\d\-\$]+')
-    job_array_template = CUnicode('#$$ -t 1-$n')
-    queue_regexp = CUnicode('#$$\W+-q\W+\$?\w+')
-    queue_template = CUnicode('#$$ -q $queue')
+    job_array_regexp = Unicode('#\$\W+\-t')
+    job_array_template = Unicode('#$ -t 1-{n}')
+    queue_regexp = Unicode('#\$\W+-q\W+\$?\w+')
+    queue_template = Unicode('#$ -q $queue')
 
 class SGEControllerLauncher(SGELauncher):
     """Launch a controller using SGE."""
 
-    batch_file_name = CUnicode(u'sge_controller', config=True)
-    default_template= CUnicode(u"""#$$ -V
-#$$ -S /bin/sh
-#$$ -N ipcontroller
-%s --log-to-file --cluster-dir $cluster_dir
+    batch_file_name = Unicode(u'sge_controller', config=True,
+        help="batch file name for the ipontroller job.")
+    default_template= Unicode(u"""#$ -V
+#$ -S /bin/sh
+#$ -N ipcontroller
+%s --log-to-file profile_dir={profile_dir}
 """%(' '.join(ipcontroller_cmd_argv)))
 
-    def start(self, cluster_dir):
-        """Start the controller by profile or cluster_dir."""
+    def start(self, profile_dir):
+        """Start the controller by profile or profile_dir."""
         self.log.info("Starting PBSControllerLauncher: %r" % self.args)
-        return super(PBSControllerLauncher, self).start(1, cluster_dir)
+        return super(SGEControllerLauncher, self).start(1, profile_dir)
 
 class SGEEngineSetLauncher(SGELauncher):
     """Launch Engines with SGE"""
-    batch_file_name = CUnicode(u'sge_engines', config=True)
-    default_template = CUnicode("""#$$ -V
-#$$ -S /bin/sh
-#$$ -N ipengine
-%s --cluster-dir $cluster_dir
+    batch_file_name = Unicode(u'sge_engines', config=True,
+        help="batch file name for the engine(s) job.")
+    default_template = Unicode("""#$ -V
+#$ -S /bin/sh
+#$ -N ipengine
+%s profile_dir={profile_dir}
 """%(' '.join(ipengine_cmd_argv)))
 
-    def start(self, n, cluster_dir):
-        """Start n engines by profile or cluster_dir."""
+    def start(self, n, profile_dir):
+        """Start n engines by profile or profile_dir."""
         self.log.info('Starting %i engines with SGEEngineSetLauncher: %r' % (n, self.args))
-        return super(SGEEngineSetLauncher, self).start(n, cluster_dir)
+        return super(SGEEngineSetLauncher, self).start(n, profile_dir)
 
 
 #-----------------------------------------------------------------------------
@@ -979,18 +1007,57 b' class SGEEngineSetLauncher(SGELauncher):'
 class IPClusterLauncher(LocalProcessLauncher):
     """Launch the ipcluster program in an external process."""
 
-    ipcluster_cmd = List(ipcluster_cmd_argv, config=True)
-    # Command line arguments to pass to ipcluster.
+    ipcluster_cmd = List(ipcluster_cmd_argv, config=True,
+        help="Popen command for ipcluster")
     ipcluster_args = List(
-        ['--clean-logs', '--log-to-file', '--log-level', str(logging.INFO)], config=True)
-    ipcluster_subcommand = Str('start')
+        ['--clean-logs', '--log-to-file', 'log_level=%i'%logging.INFO], config=True,
+        help="Command line arguments to pass to ipcluster.")
+    ipcluster_subcommand = Unicode('start')
     ipcluster_n = Int(2)
 
     def find_args(self):
-        return self.ipcluster_cmd + [self.ipcluster_subcommand] + \
-            ['-n', repr(self.ipcluster_n)] + self.ipcluster_args
+        return self.ipcluster_cmd + ['--'+self.ipcluster_subcommand] + \
+            ['n=%i'%self.ipcluster_n] + self.ipcluster_args
 
     def start(self):
         self.log.info("Starting ipcluster: %r" % self.args)
         return super(IPClusterLauncher, self).start()
 
+#-----------------------------------------------------------------------------
+# Collections of launchers
+#-----------------------------------------------------------------------------
+
+local_launchers = [
+    LocalControllerLauncher,
+    LocalEngineLauncher,
+    LocalEngineSetLauncher,
+]
+mpi_launchers = [
+    MPIExecLauncher,
+    MPIExecControllerLauncher,
+    MPIExecEngineSetLauncher,
+]
+ssh_launchers = [
+    SSHLauncher,
+    SSHControllerLauncher,
+    SSHEngineLauncher,
+    SSHEngineSetLauncher,
+]
+winhpc_launchers = [
+    WindowsHPCLauncher,
+    WindowsHPCControllerLauncher,
+    WindowsHPCEngineSetLauncher,
+]
+pbs_launchers = [
+    PBSLauncher,
+    PBSControllerLauncher,
+    PBSEngineSetLauncher,
+]
+sge_launchers = [
+    SGELauncher,
+    SGEControllerLauncher,
+    SGEEngineSetLauncher,
+]
+all_launchers = local_launchers + mpi_launchers + ssh_launchers + winhpc_launchers\
+                + pbs_launchers + sge_launchers
+

IPython/parallel/apps/logwatcher.py

@@ -1,5 +1,12 b''
 #!/usr/bin/env python
-"""A simple logger object that consolidates messages incoming from ipcluster processes."""
+"""
+A simple logger object that consolidates messages incoming from ipcluster processes.
+
+Authors:
+
+* MinRK
+
+"""
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2011  The IPython Development Team
@@ -19,29 +26,35 b' import sys'
 import zmq
 from zmq.eventloop import ioloop, zmqstream
 
-from IPython.utils.traitlets import Int, Str, Instance, List
-
-from IPython.parallel.factory import LoggingFactory
+from IPython.config.configurable import LoggingConfigurable
+from IPython.utils.traitlets import Int, Unicode, Instance, List
 
 #-----------------------------------------------------------------------------
 # Classes
 #-----------------------------------------------------------------------------
 
 
-class LogWatcher(LoggingFactory):
+class LogWatcher(LoggingConfigurable):
     """A simple class that receives messages on a SUB socket, as published
     by subclasses of `zmq.log.handlers.PUBHandler`, and logs them itself.
     
     This can subscribe to multiple topics, but defaults to all topics.
     """
+    
     # configurables
-    topics = List([''], config=True)
-    url = Str('tcp://127.0.0.1:20202', config=True)
+    topics = List([''], config=True,
+        help="The ZMQ topics to subscribe to. Default is to subscribe to all messages")
+    url = Unicode('tcp://127.0.0.1:20202', config=True,
+        help="ZMQ url on which to listen for log messages")
     
     # internals
-    context = Instance(zmq.Context, (), {})
     stream = Instance('zmq.eventloop.zmqstream.ZMQStream')
-    loop = Instance('zmq.eventloop.ioloop.IOLoop')
+    
+    context = Instance(zmq.Context)
+    def _context_default(self):
+        return zmq.Context.instance()
+    
+    loop = Instance(zmq.eventloop.ioloop.IOLoop)
     def _loop_default(self):
         return ioloop.IOLoop.instance()
     
@@ -62,9 +75,13 b' class LogWatcher(LoggingFactory):'
     def subscribe(self):
         """Update our SUB socket's subscriptions."""
         self.stream.setsockopt(zmq.UNSUBSCRIBE, '')
-        for topic in self.topics:
-            self.log.debug("Subscribing to: %r"%topic)
-            self.stream.setsockopt(zmq.SUBSCRIBE, topic)
+        if '' in self.topics:
+            self.log.debug("Subscribing to: everything")
+            self.stream.setsockopt(zmq.SUBSCRIBE, '')
+        else:
+            for topic in self.topics:
+                self.log.debug("Subscribing to: %r"%(topic))
+                self.stream.setsockopt(zmq.SUBSCRIBE, topic)
     
     def _extract_level(self, topic_str):
         """Turn 'engine.0.INFO.extra' into (logging.INFO, 'engine.0.extra')"""
@@ -94,5 +111,5 b' class LogWatcher(LoggingFactory):'
             level,topic = self._extract_level(topic)
             if msg[-1] == '\n':
                 msg = msg[:-1]
-            logging.log(level, "[%s] %s" % (topic, msg))
+            self.log.log(level, "[%s] %s" % (topic, msg))
 

IPython/parallel/apps/win32support.py

@@ -1,7 +1,13 b''
 #!/usr/bin/env python
 """Utility for forwarding file read events over a zmq socket.
 
-This is necessary because select on Windows only supports"""
+This is necessary because select on Windows only supports sockets, not FDs.
+
+Authors:
+
+* MinRK
+
+"""
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2011  The IPython Development Team
@@ -31,7 +37,7 b' class ForwarderThread(Thread):'
         self.fd = fd
 
     def run(self):
-        """loop through lines in self.fd, and send them over self.sock"""
+        """Loop through lines in self.fd, and send them over self.sock."""
         line = self.fd.readline()
         # allow for files opened in unicode mode
         if isinstance(line, unicode):
@@ -46,7 +52,7 b' class ForwarderThread(Thread):'
         self.sock.close()
 
 def forward_read_events(fd, context=None):
-    """forward read events from an FD over a socket.
+    """Forward read events from an FD over a socket.
 
     This method wraps a file in a socket pair, so it can
     be polled for read events by select (specifically zmq.eventloop.ioloop)
@@ -64,4 +70,4 b' def forward_read_events(fd, context=None):'
     return pull
 
 
-__all__ = ['forward_read_events']
  No newline at end of file
+__all__ = ['forward_read_events']

IPython/parallel/apps/winhpcjob.py

@@ -3,10 +3,16 b''
 """
 Job and task components for writing .xml files that the Windows HPC Server 
 2008 can use to start jobs.
+
+Authors:
+
+* Brian Granger
+* MinRK
+
 """
 
 #-----------------------------------------------------------------------------
-#  Copyright (C) 2008-2009  The IPython Development Team
+#  Copyright (C) 2008-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
@@ -24,8 +30,8 b' from xml.etree import ElementTree as ET'
 
 from IPython.config.configurable import Configurable
 from IPython.utils.traitlets import (
-    Str, Int, List, Instance,
-    Enum, Bool, CStr
+    Unicode, Int, List, Instance,
+    Enum, Bool
 )
 
 #-----------------------------------------------------------------------------
@@ -74,27 +80,27 b' def find_username():'
 
 class WinHPCJob(Configurable):
 
-    job_id = Str('')
-    job_name = Str('MyJob', config=True)
+    job_id = Unicode('')
+    job_name = Unicode('MyJob', config=True)
     min_cores = Int(1, config=True)
     max_cores = Int(1, config=True)
     min_sockets = Int(1, config=True)
     max_sockets = Int(1, config=True)
     min_nodes = Int(1, config=True)
     max_nodes = Int(1, config=True)
-    unit_type = Str("Core", config=True)
+    unit_type = Unicode("Core", config=True)
     auto_calculate_min = Bool(True, config=True)
     auto_calculate_max = Bool(True, config=True)
     run_until_canceled = Bool(False, config=True)
     is_exclusive = Bool(False, config=True)
-    username = Str(find_username(), config=True)
-    job_type = Str('Batch', config=True)
+    username = Unicode(find_username(), config=True)
+    job_type = Unicode('Batch', config=True)
     priority = Enum(('Lowest','BelowNormal','Normal','AboveNormal','Highest'),
         default_value='Highest', config=True)
-    requested_nodes = Str('', config=True)
-    project = Str('IPython', config=True)
-    xmlns = Str('http://schemas.microsoft.com/HPCS2008/scheduler/')
-    version = Str("2.000")
+    requested_nodes = Unicode('', config=True)
+    project = Unicode('IPython', config=True)
+    xmlns = Unicode('http://schemas.microsoft.com/HPCS2008/scheduler/')
+    version = Unicode("2.000")
     tasks = List([])
 
     @property
@@ -165,21 +171,21 b' class WinHPCJob(Configurable):'
 
 class WinHPCTask(Configurable):
 
-    task_id = Str('')
-    task_name = Str('')
-    version = Str("2.000")
+    task_id = Unicode('')
+    task_name = Unicode('')
+    version = Unicode("2.000")
     min_cores = Int(1, config=True)
     max_cores = Int(1, config=True)
     min_sockets = Int(1, config=True)
     max_sockets = Int(1, config=True)
     min_nodes = Int(1, config=True)
     max_nodes = Int(1, config=True)
-    unit_type = Str("Core", config=True)
-    command_line = CStr('', config=True)
-    work_directory = CStr('', config=True)
+    unit_type = Unicode("Core", config=True)
+    command_line = Unicode('', config=True)
+    work_directory = Unicode('', config=True)
     is_rerunnaable = Bool(True, config=True)
-    std_out_file_path = CStr('', config=True)
-    std_err_file_path = CStr('', config=True)
+    std_out_file_path = Unicode('', config=True)
+    std_err_file_path = Unicode('', config=True)
     is_parametric = Bool(False, config=True)
     environment_variables = Instance(dict, args=(), config=True)
 
@@ -223,41 +229,41 b' class WinHPCTask(Configurable):'
 # By declaring these, we can configure the controller and engine separately!
 
 class IPControllerJob(WinHPCJob):
-    job_name = Str('IPController', config=False)
+    job_name = Unicode('IPController', config=False)
     is_exclusive = Bool(False, config=True)
-    username = Str(find_username(), config=True)
+    username = Unicode(find_username(), config=True)
     priority = Enum(('Lowest','BelowNormal','Normal','AboveNormal','Highest'),
         default_value='Highest', config=True)
-    requested_nodes = Str('', config=True)
-    project = Str('IPython', config=True)
+    requested_nodes = Unicode('', config=True)
+    project = Unicode('IPython', config=True)
 
 
 class IPEngineSetJob(WinHPCJob):
-    job_name = Str('IPEngineSet', config=False)
+    job_name = Unicode('IPEngineSet', config=False)
     is_exclusive = Bool(False, config=True)
-    username = Str(find_username(), config=True)
+    username = Unicode(find_username(), config=True)
     priority = Enum(('Lowest','BelowNormal','Normal','AboveNormal','Highest'),
         default_value='Highest', config=True)
-    requested_nodes = Str('', config=True)
-    project = Str('IPython', config=True)
+    requested_nodes = Unicode('', config=True)
+    project = Unicode('IPython', config=True)
 
 
 class IPControllerTask(WinHPCTask):
 
-    task_name = Str('IPController', config=True)
+    task_name = Unicode('IPController', config=True)
     controller_cmd = List(['ipcontroller.exe'], config=True)
-    controller_args = List(['--log-to-file', '--log-level', '40'], config=True)
+    controller_args = List(['--log-to-file', 'log-level=40'], config=True)
     # I don't want these to be configurable
-    std_out_file_path = CStr('', config=False)
-    std_err_file_path = CStr('', config=False)
+    std_out_file_path = Unicode('', config=False)
+    std_err_file_path = Unicode('', config=False)
     min_cores = Int(1, config=False)
     max_cores = Int(1, config=False)
     min_sockets = Int(1, config=False)
     max_sockets = Int(1, config=False)
     min_nodes = Int(1, config=False)
     max_nodes = Int(1, config=False)
-    unit_type = Str("Core", config=False)
-    work_directory = CStr('', config=False)
+    unit_type = Unicode("Core", config=False)
+    work_directory = Unicode('', config=False)
 
     def __init__(self, config=None):
         super(IPControllerTask, self).__init__(config=config)
@@ -272,20 +278,20 b' class IPControllerTask(WinHPCTask):'
 
 class IPEngineTask(WinHPCTask):
 
-    task_name = Str('IPEngine', config=True)
+    task_name = Unicode('IPEngine', config=True)
     engine_cmd = List(['ipengine.exe'], config=True)
-    engine_args = List(['--log-to-file', '--log-level', '40'], config=True)
+    engine_args = List(['--log-to-file', 'log_level=40'], config=True)
     # I don't want these to be configurable
-    std_out_file_path = CStr('', config=False)
-    std_err_file_path = CStr('', config=False)
+    std_out_file_path = Unicode('', config=False)
+    std_err_file_path = Unicode('', config=False)
     min_cores = Int(1, config=False)
     max_cores = Int(1, config=False)
     min_sockets = Int(1, config=False)
     max_sockets = Int(1, config=False)
     min_nodes = Int(1, config=False)
     max_nodes = Int(1, config=False)
-    unit_type = Str("Core", config=False)
-    work_directory = CStr('', config=False)
+    unit_type = Unicode("Core", config=False)
+    work_directory = Unicode('', config=False)
 
     def __init__(self, config=None):
         super(IPEngineTask,self).__init__(config=config)

IPython/parallel/client/asyncresult.py

@@ -1,4 +1,9 b''
-"""AsyncResult objects for the client"""
+"""AsyncResult objects for the client
+
+Authors:
+
+* MinRK
+"""
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2010-2011  The IPython Development Team
 #

IPython/parallel/client/client.py

@@ -1,6 +1,11 b''
-"""A semi-synchronous Client for the ZMQ cluster"""
+"""A semi-synchronous Client for the ZMQ cluster
+
+Authors:
+
+* MinRK
+"""
 #-----------------------------------------------------------------------------
-#  Copyright (C) 2010  The IPython Development Team
+#  Copyright (C) 2010-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
@@ -23,18 +28,20 b' pjoin = os.path.join'
 import zmq
 # from zmq.eventloop import ioloop, zmqstream
 
+from IPython.utils.jsonutil import rekey
 from IPython.utils.path import get_ipython_dir
-from IPython.utils.traitlets import (HasTraits, Int, Instance, CUnicode, 
-                                    Dict, List, Bool, Str, Set)
+from IPython.utils.traitlets import (HasTraits, Int, Instance, Unicode,
+                                    Dict, List, Bool, Set)
 from IPython.external.decorator import decorator
 from IPython.external.ssh import tunnel
 
 from IPython.parallel import error
-from IPython.parallel import streamsession as ss
 from IPython.parallel import util
 
+from IPython.zmq.session import Session, Message
+
 from .asyncresult import AsyncResult, AsyncHubResult
-from IPython.parallel.apps.clusterdir import ClusterDir, ClusterDirError
+from IPython.core.profiledir import ProfileDir, ProfileDirError
 from .view import DirectView, LoadBalancedView
 
 #--------------------------------------------------------------------------
@@ -119,11 +126,27 b' class Client(HasTraits):'
         [Default: 'default']
     context : zmq.Context
         Pass an existing zmq.Context instance, otherwise the client will create its own.
-    username : bytes
-        set username to be passed to the Session object
     debug : bool
         flag for lots of message printing for debug purposes
+    timeout : int/float 
+        time (in seconds) to wait for connection replies from the Hub
+        [Default: 10]
  
+    #-------------- session related args ----------------
+    
+    config : Config object
+        If specified, this will be relayed to the Session for configuration
+    username : str
+        set username for the session object
+    packer : str (import_string) or callable
+        Can be either the simple keyword 'json' or 'pickle', or an import_string to a
+        function to serialize messages. Must support same input as
+        JSON, and output must be bytes.
+        You can pass a callable directly as `pack`
+    unpacker : str (import_string) or callable
+        The inverse of packer.  Only necessary if packer is specified as *not* one
+        of 'json' or 'pickle'.
+    
     #-------------- ssh related args ----------------
     # These are args for configuring the ssh tunnel to be used
     # credentials are used to forward connections over ssh to the Controller
@@ -149,9 +172,10 b' class Client(HasTraits):'
     
     ------- exec authentication args -------
     If even localhost is untrusted, you can have some protection against
-    unauthorized execution by using a key.  Messages are still sent
-    as cleartext, so if someone can snoop your loopback traffic this will
-    not help against malicious attacks.
+    unauthorized execution by signing messages with HMAC digests.  
+    Messages are still sent as cleartext, so if someone can snoop your 
+    loopback traffic this will not protect your privacy, but will prevent
+    unauthorized execution.
     
     exec_key : str
         an authentication key or file containing a key
@@ -213,7 +237,7 b' class Client(HasTraits):'
     metadata = Instance('collections.defaultdict', (Metadata,))
     history = List()
     debug = Bool(False)
-    profile=CUnicode('default')
+    profile=Unicode('default')
     
     _outstanding_dict = Instance('collections.defaultdict', (set,))
     _ids = List()
@@ -229,15 +253,15 b' class Client(HasTraits):'
     _notification_socket=Instance('zmq.Socket')
     _mux_socket=Instance('zmq.Socket')
     _task_socket=Instance('zmq.Socket')
-    _task_scheme=Str()
+    _task_scheme=Unicode()
     _closed = False
     _ignored_control_replies=Int(0)
     _ignored_hub_replies=Int(0)
     
-    def __init__(self, url_or_file=None, profile='default', cluster_dir=None, ipython_dir=None,
-            context=None, username=None, debug=False, exec_key=None,
+    def __init__(self, url_or_file=None, profile='default', profile_dir=None, ipython_dir=None,
+            context=None, debug=False, exec_key=None,
             sshserver=None, sshkey=None, password=None, paramiko=None,
-            timeout=10
+            timeout=10, **extra_args
             ):
         super(Client, self).__init__(debug=debug, profile=profile)
         if context is None:
@@ -245,7 +269,7 b' class Client(HasTraits):'
         self._context = context
             
         
-        self._setup_cluster_dir(profile, cluster_dir, ipython_dir)
+        self._setup_profile_dir(profile, profile_dir, ipython_dir)
         if self._cd is not None:
             if url_or_file is None:
                 url_or_file = pjoin(self._cd.security_dir, 'ipcontroller-client.json')
@@ -288,15 +312,15 b' class Client(HasTraits):'
             else:
                 password = getpass("SSH Password for %s: "%sshserver)
         ssh_kwargs = dict(keyfile=sshkey, password=password, paramiko=paramiko)
-        if exec_key is not None and os.path.isfile(exec_key):
-            arg = 'keyfile'
-        else:
-            arg = 'key'
-        key_arg = {arg:exec_key}
-        if username is None:
-            self.session = ss.StreamSession(**key_arg)
-        else:
-            self.session = ss.StreamSession(username, **key_arg)
+        
+        # configure and construct the session
+        if exec_key is not None:
+            if os.path.isfile(exec_key):
+                extra_args['keyfile'] = exec_key
+            else:
+                extra_args['key'] = exec_key
+        self.session = Session(**extra_args)
+        
         self._query_socket = self._context.socket(zmq.XREQ)
         self._query_socket.setsockopt(zmq.IDENTITY, self.session.session)
         if self._ssh:
@@ -318,21 +342,21 b' class Client(HasTraits):'
         """cleanup sockets, but _not_ context."""
         self.close()
     
-    def _setup_cluster_dir(self, profile, cluster_dir, ipython_dir):
+    def _setup_profile_dir(self, profile, profile_dir, ipython_dir):
         if ipython_dir is None:
             ipython_dir = get_ipython_dir()
-        if cluster_dir is not None:
+        if profile_dir is not None:
             try:
-                self._cd = ClusterDir.find_cluster_dir(cluster_dir)
+                self._cd = ProfileDir.find_profile_dir(profile_dir)
                 return
-            except ClusterDirError:
+            except ProfileDirError:
                 pass
         elif profile is not None:
             try:
-                self._cd = ClusterDir.find_cluster_dir_by_profile(
+                self._cd = ProfileDir.find_profile_dir_by_name(
                     ipython_dir, profile)
                 return
-            except ClusterDirError:
+            except ProfileDirError:
                 pass
         self._cd = None
     
@@ -416,7 +440,7 b' class Client(HasTraits):'
         idents,msg = self.session.recv(self._query_socket,mode=0)
         if self.debug:
             pprint(msg)
-        msg = ss.Message(msg)
+        msg = Message(msg)
         content = msg.content
         self._config['registration'] = dict(content)
         if content.status == 'ok':
@@ -478,11 +502,11 b' class Client(HasTraits):'
             md['engine_id'] = self._engines.get(md['engine_uuid'], None)
         
         if 'date' in parent:
-            md['submitted'] = datetime.strptime(parent['date'], util.ISO8601)
+            md['submitted'] = parent['date']
         if 'started' in header:
-            md['started'] = datetime.strptime(header['started'], util.ISO8601)
+            md['started'] = header['started']
         if 'date' in header:
-            md['completed'] = datetime.strptime(header['date'], util.ISO8601)
+            md['completed'] = header['date']
         return md
     
     def _register_engine(self, msg):
@@ -528,7 +552,7 b' class Client(HasTraits):'
             header = {}
             parent['msg_id'] = msg_id
             header['engine'] = uuid
-            header['date'] = datetime.now().strftime(util.ISO8601)
+            header['date'] = datetime.now()
             msg = dict(parent_header=parent, header=header, content=content)
             self._handle_apply_reply(msg)
     
@@ -589,33 +613,31 b' class Client(HasTraits):'
     def _flush_notifications(self):
         """Flush notifications of engine registrations waiting
         in ZMQ queue."""
-        msg = self.session.recv(self._notification_socket, mode=zmq.NOBLOCK)
+        idents,msg = self.session.recv(self._notification_socket, mode=zmq.NOBLOCK)
         while msg is not None:
             if self.debug:
                 pprint(msg)
-            msg = msg[-1]
             msg_type = msg['msg_type']
             handler = self._notification_handlers.get(msg_type, None)
             if handler is None:
                 raise Exception("Unhandled message type: %s"%msg.msg_type)
             else:
                 handler(msg)
-            msg = self.session.recv(self._notification_socket, mode=zmq.NOBLOCK)
+            idents,msg = self.session.recv(self._notification_socket, mode=zmq.NOBLOCK)
     
     def _flush_results(self, sock):
         """Flush task or queue results waiting in ZMQ queue."""
-        msg = self.session.recv(sock, mode=zmq.NOBLOCK)
+        idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK)
         while msg is not None:
             if self.debug:
                 pprint(msg)
-            msg = msg[-1]
             msg_type = msg['msg_type']
             handler = self._queue_handlers.get(msg_type, None)
             if handler is None:
                 raise Exception("Unhandled message type: %s"%msg.msg_type)
             else:
                 handler(msg)
-            msg = self.session.recv(sock, mode=zmq.NOBLOCK)
+            idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK)
     
     def _flush_control(self, sock):
         """Flush replies from the control channel waiting
@@ -624,12 +646,12 b' class Client(HasTraits):'
         Currently: ignore them."""
         if self._ignored_control_replies <= 0:
             return
-        msg = self.session.recv(sock, mode=zmq.NOBLOCK)
+        idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK)
         while msg is not None:
             self._ignored_control_replies -= 1
             if self.debug:
                 pprint(msg)
-            msg = self.session.recv(sock, mode=zmq.NOBLOCK)
+            idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK)
     
     def _flush_ignored_control(self):
         """flush ignored control replies"""
@@ -638,19 +660,18 b' class Client(HasTraits):'
             self._ignored_control_replies -= 1
     
     def _flush_ignored_hub_replies(self):
-        msg = self.session.recv(self._query_socket, mode=zmq.NOBLOCK)
+        ident,msg = self.session.recv(self._query_socket, mode=zmq.NOBLOCK)
         while msg is not None:
-            msg = self.session.recv(self._query_socket, mode=zmq.NOBLOCK)
+            ident,msg = self.session.recv(self._query_socket, mode=zmq.NOBLOCK)
     
     def _flush_iopub(self, sock):
         """Flush replies from the iopub channel waiting
         in the ZMQ queue.
         """
-        msg = self.session.recv(sock, mode=zmq.NOBLOCK)
+        idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK)
         while msg is not None:
             if self.debug:
                 pprint(msg)
-            msg = msg[-1]
             parent = msg['parent_header']
             msg_id = parent['msg_id']
             content = msg['content']
@@ -674,7 +695,7 b' class Client(HasTraits):'
             # reduntant?
             self.metadata[msg_id] = md
             
-            msg = self.session.recv(sock, mode=zmq.NOBLOCK)
+            idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK)
     
     #--------------------------------------------------------------------------
     # len, getitem
@@ -1172,6 +1193,7 b' class Client(HasTraits):'
         failures = []
         # load cached results into result:
         content.update(local_results)
+        
         # update cache with results:
         for msg_id in sorted(theids):
             if msg_id in content['completed']:
@@ -1226,7 +1248,7 b' class Client(HasTraits):'
         status = content.pop('status')
         if status != 'ok':
             raise self._unwrap_exception(content)
-        content = util.rekey(content)
+        content = rekey(content)
         if isinstance(targets, int):
             return content[targets]
         else:
@@ -1332,6 +1354,7 b' class Client(HasTraits):'
             raise self._unwrap_exception(content)
         
         records = content['records']
+        
         buffer_lens = content['buffer_lens']
         result_buffer_lens = content['result_buffer_lens']
         buffers = msg['buffers']
@@ -1345,11 +1368,6 b' class Client(HasTraits):'
             if has_rbufs:
                 blen = result_buffer_lens[i]
                 rec['result_buffers'], buffers = buffers[:blen],buffers[blen:]
-            # turn timestamps back into times
-            for key in 'submitted started completed resubmitted'.split():
-                maybedate = rec.get(key, None)
-                if maybedate and util.ISO8601_RE.match(maybedate):
-                    rec[key] = datetime.strptime(maybedate, util.ISO8601)
             
         return records
 

IPython/parallel/client/map.py

@@ -4,12 +4,19 b''
 
 Scattering consists of partitioning a sequence and sending the various
 pieces to individual nodes in a cluster.
+
+
+Authors:
+
+* Brian Granger
+* MinRK
+
 """
 
 __docformat__ = "restructuredtext en"
 
 #-------------------------------------------------------------------------------
-#  Copyright (C) 2008  The IPython Development Team
+#  Copyright (C) 2008-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.

IPython/parallel/client/remotefunction.py

@@ -1,6 +1,12 b''
-"""Remote Functions and decorators for Views."""
+"""Remote Functions and decorators for Views.
+
+Authors:
+
+* Brian Granger
+* Min RK
+"""
 #-----------------------------------------------------------------------------
-#  Copyright (C) 2010  The IPython Development Team
+#  Copyright (C) 2010-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.

IPython/parallel/client/view.py

@@ -1,6 +1,11 b''
-"""Views of remote engines."""
+"""Views of remote engines.
+
+Authors:
+
+* Min RK
+"""
 #-----------------------------------------------------------------------------
-#  Copyright (C) 2010  The IPython Development Team
+#  Copyright (C) 2010-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.

IPython/parallel/controller/dependency.py

@@ -1,4 +1,9 b''
-"""Dependency utilities"""
+"""Dependency utilities
+
+Authors:
+
+* Min RK
+"""
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2010-2011  The IPython Development Team
 #

IPython/parallel/controller/dictdb.py

@@ -1,6 +1,11 b''
 """A Task logger that presents our DB interface, 
 but exists entirely in memory and implemented with dicts.
 
+Authors:
+
+* Min RK
+
+
 TaskRecords are dicts of the form:
 {
     'msg_id' : str(uuid),
@@ -35,7 +40,7 b' We support a subset of mongodb operators:'
     $lt,$gt,$lte,$gte,$ne,$in,$nin,$all,$mod,$exists
 """
 #-----------------------------------------------------------------------------
-#  Copyright (C) 2010  The IPython Development Team
+#  Copyright (C) 2010-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
@@ -44,9 +49,9 b' We support a subset of mongodb operators:'
 
 from datetime import datetime
 
-from IPython.config.configurable import Configurable
+from IPython.config.configurable import LoggingConfigurable
 
-from IPython.utils.traitlets import Dict, CUnicode
+from IPython.utils.traitlets import Dict, Unicode, Instance
 
 filters = {
  '$lt' : lambda a,b: a < b,
@@ -79,10 +84,10 b' class CompositeFilter(object):'
                 return False
         return True
 
-class BaseDB(Configurable):
+class BaseDB(LoggingConfigurable):
     """Empty Parent class so traitlets work on DB."""
     # base configurable traits:
-    session = CUnicode("")
+    session = Unicode("")
 
 class DictDB(BaseDB):
     """Basic in-memory dict-based object for saving Task Records.

IPython/parallel/controller/heartmonitor.py

@@ -2,6 +2,10 b''
 """
 A multi-heart Heartbeat system using PUB and XREP sockets. pings are sent out on the PUB,
 and hearts are tracked based on their XREQ identities.
+
+Authors:
+
+* Min RK
 """
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2010-2011  The IPython Development Team
@@ -15,11 +19,11 b' import time'
 import uuid
 
 import zmq
-from zmq.devices import ProcessDevice, ThreadDevice
+from zmq.devices import ThreadDevice
 from zmq.eventloop import ioloop, zmqstream
 
-from IPython.utils.traitlets import Set, Instance, CFloat, Bool
-from IPython.parallel.factory import LoggingFactory
+from IPython.config.configurable import LoggingConfigurable
+from IPython.utils.traitlets import Set, Instance, CFloat
 
 class Heart(object):
     """A basic heart object for responding to a HeartMonitor.
@@ -47,20 +51,22 b' class Heart(object):'
     def start(self):
         return self.device.start()
         
-class HeartMonitor(LoggingFactory):
+class HeartMonitor(LoggingConfigurable):
     """A basic HeartMonitor class
     pingstream: a PUB stream
     pongstream: an XREP stream
     period: the period of the heartbeat in milliseconds"""
     
-    period=CFloat(1000, config=True) # in milliseconds
+    period=CFloat(1000, config=True,
+        help='The frequency at which the Hub pings the engines for heartbeats '
+        ' (in ms) [default: 100]',
+    )
     
     pingstream=Instance('zmq.eventloop.zmqstream.ZMQStream')
     pongstream=Instance('zmq.eventloop.zmqstream.ZMQStream')
     loop = Instance('zmq.eventloop.ioloop.IOLoop')
     def _loop_default(self):
         return ioloop.IOLoop.instance()
-    debug=Bool(False)
     
     # not settable:
     hearts=Set()

IPython/parallel/controller/hub.py

@@ -2,6 +2,10 b''
 """The IPython Controller Hub with 0MQ
 This is the master object that handles connections from engines and clients,
 and monitors traffic through the various queues.
+
+Authors:
+
+* Min RK
 """
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2010  The IPython Development Team
@@ -25,10 +29,14 b' from zmq.eventloop.zmqstream import ZMQStream'
 
 # internal:
 from IPython.utils.importstring import import_item
-from IPython.utils.traitlets import HasTraits, Instance, Int, CStr, Str, Dict, Set, List, Bool
+from IPython.utils.traitlets import (
+        HasTraits, Instance, Int, Unicode, Dict, Set, Tuple, CStr
+        )
 
 from IPython.parallel import error, util
-from IPython.parallel.factory import RegistrationFactory, LoggingFactory
+from IPython.parallel.factory import RegistrationFactory
+
+from IPython.zmq.session import SessionFactory
 
 from .heartmonitor import HeartMonitor
 
@@ -75,7 +83,7 b' def init_record(msg):'
         'header' : header,
         'content': msg['content'],
         'buffers': msg['buffers'],
-        'submitted': datetime.strptime(header['date'], util.ISO8601),
+        'submitted': header['date'],
         'client_uuid' : None,
         'engine_uuid' : None,
         'started': None,
@@ -103,68 +111,80 b' class EngineConnector(HasTraits):'
     heartbeat (str): identity of heartbeat XREQ socket
     """
     id=Int(0)
-    queue=Str()
-    control=Str()
-    registration=Str()
-    heartbeat=Str()
+    queue=CStr()
+    control=CStr()
+    registration=CStr()
+    heartbeat=CStr()
     pending=Set()
 
 class HubFactory(RegistrationFactory):
     """The Configurable for setting up a Hub."""
     
-    # name of a scheduler scheme
-    scheme = Str('leastload', config=True)
-    
     # port-pairs for monitoredqueues:
-    hb = Instance(list, config=True)
+    hb = Tuple(Int,Int,config=True,
+        help="""XREQ/SUB Port pair for Engine heartbeats""")
     def _hb_default(self):
-        return util.select_random_ports(2)
+        return tuple(util.select_random_ports(2))
+
+    mux = Tuple(Int,Int,config=True,
+        help="""Engine/Client Port pair for MUX queue""")
     
-    mux = Instance(list, config=True)
     def _mux_default(self):
-        return util.select_random_ports(2)
+        return tuple(util.select_random_ports(2))
     
-    task = Instance(list, config=True)
+    task = Tuple(Int,Int,config=True,
+        help="""Engine/Client Port pair for Task queue""")
     def _task_default(self):
-        return util.select_random_ports(2)
+        return tuple(util.select_random_ports(2))
+
+    control = Tuple(Int,Int,config=True,
+        help="""Engine/Client Port pair for Control queue""")
     
-    control = Instance(list, config=True)
     def _control_default(self):
-        return util.select_random_ports(2)
+        return tuple(util.select_random_ports(2))
+
+    iopub = Tuple(Int,Int,config=True,
+        help="""Engine/Client Port pair for IOPub relay""")
     
-    iopub = Instance(list, config=True)
     def _iopub_default(self):
-        return util.select_random_ports(2)
+        return tuple(util.select_random_ports(2))
     
     # single ports:
-    mon_port = Instance(int, config=True)
+    mon_port = Int(config=True,
+        help="""Monitor (SUB) port for queue traffic""")
+
     def _mon_port_default(self):
         return util.select_random_ports(1)[0]
     
-    notifier_port = Instance(int, config=True)
+    notifier_port = Int(config=True,
+        help="""PUB port for sending engine status notifications""")
+
     def _notifier_port_default(self):
         return util.select_random_ports(1)[0]
     
-    ping = Int(1000, config=True) # ping frequency
+    engine_ip = Unicode('127.0.0.1', config=True,
+        help="IP on which to listen for engine connections. [default: loopback]")
+    engine_transport = Unicode('tcp', config=True,
+        help="0MQ transport for engine connections. [default: tcp]")
     
-    engine_ip = CStr('127.0.0.1', config=True)
-    engine_transport = CStr('tcp', config=True)
+    client_ip = Unicode('127.0.0.1', config=True,
+        help="IP on which to listen for client connections. [default: loopback]")
+    client_transport = Unicode('tcp', config=True,
+        help="0MQ transport for client connections. [default : tcp]")
     
-    client_ip = CStr('127.0.0.1', config=True)
-    client_transport = CStr('tcp', config=True)
+    monitor_ip = Unicode('127.0.0.1', config=True,
+        help="IP on which to listen for monitor messages. [default: loopback]")
+    monitor_transport = Unicode('tcp', config=True,
+        help="0MQ transport for monitor messages. [default : tcp]")
     
-    monitor_ip = CStr('127.0.0.1', config=True)
-    monitor_transport = CStr('tcp', config=True)
+    monitor_url = Unicode('')
     
-    monitor_url = CStr('')
-    
-    db_class = CStr('IPython.parallel.controller.dictdb.DictDB', config=True)
+    db_class = Unicode('IPython.parallel.controller.dictdb.DictDB', config=True,
+        help="""The class to use for the DB backend""")
     
     # not configurable
     db = Instance('IPython.parallel.controller.dictdb.BaseDB')
     heartmonitor = Instance('IPython.parallel.controller.heartmonitor.HeartMonitor')
-    subconstructors = List()
-    _constructed = Bool(False)
     
     def _ip_changed(self, name, old, new):
         self.engine_ip = new
@@ -184,26 +204,16 b' class HubFactory(RegistrationFactory):'
     def __init__(self, **kwargs):
         super(HubFactory, self).__init__(**kwargs)
         self._update_monitor_url()
-        # self.on_trait_change(self._sync_ips, 'ip')
-        # self.on_trait_change(self._sync_transports, 'transport')
-        self.subconstructors.append(self.construct_hub)
     
     
     def construct(self):
-        assert not self._constructed, "already constructed!"
-        
-        for subc in self.subconstructors:
-            subc()
-        
-        self._constructed = True
-        
+        self.init_hub()
     
     def start(self):
-        assert self._constructed, "must be constructed by self.construct() first!"
         self.heartmonitor.start()
         self.log.info("Heartmonitor started")
     
-    def construct_hub(self):
+    def init_hub(self):
         """construct"""
         client_iface = "%s://%s:"%(self.client_transport, self.client_ip) + "%i"
         engine_iface = "%s://%s:"%(self.engine_transport, self.engine_ip) + "%i"
@@ -226,8 +236,10 b' class HubFactory(RegistrationFactory):'
         hpub.bind(engine_iface % self.hb[0])
         hrep = ctx.socket(zmq.XREP)
         hrep.bind(engine_iface % self.hb[1])
-        self.heartmonitor = HeartMonitor(loop=loop, pingstream=ZMQStream(hpub,loop), pongstream=ZMQStream(hrep,loop), 
-                                period=self.ping, logname=self.log.name)
+        self.heartmonitor = HeartMonitor(loop=loop, config=self.config, log=self.log,
+                                pingstream=ZMQStream(hpub,loop),
+                                pongstream=ZMQStream(hrep,loop)
+                            )
 
         ### Client connections ###
         # Notifier socket
@@ -246,9 +258,14 b' class HubFactory(RegistrationFactory):'
         # connect the db
         self.log.info('Hub using DB backend: %r'%(self.db_class.split()[-1]))
         # cdir = self.config.Global.cluster_dir
-        self.db = import_item(self.db_class)(session=self.session.session, config=self.config)
+        self.db = import_item(str(self.db_class))(session=self.session.session, 
+                                            config=self.config, log=self.log)
         time.sleep(.25)
-
+        try:
+            scheme = self.config.TaskScheduler.scheme_name
+        except AttributeError:
+            from .scheduler import TaskScheduler
+            scheme = TaskScheduler.scheme_name.get_default_value()
         # build connection dicts
         self.engine_info = {
             'control' : engine_iface%self.control[1],
@@ -262,7 +279,7 b' class HubFactory(RegistrationFactory):'
         self.client_info = {
             'control' : client_iface%self.control[0],
             'mux': client_iface%self.mux[0],
-            'task' : (self.scheme, client_iface%self.task[0]),
+            'task' : (scheme, client_iface%self.task[0]),
             'iopub' : client_iface%self.iopub[0],
             'notification': client_iface%self.notifier_port
             }
@@ -278,16 +295,16 b' class HubFactory(RegistrationFactory):'
         self.hub = Hub(loop=loop, session=self.session, monitor=sub, heartmonitor=self.heartmonitor,
                 query=q, notifier=n, resubmit=r, db=self.db,
                 engine_info=self.engine_info, client_info=self.client_info,
-                logname=self.log.name)
+                log=self.log)
     
 
-class Hub(LoggingFactory):
+class Hub(SessionFactory):
     """The IPython Controller Hub with 0MQ connections
     
     Parameters
     ==========
     loop: zmq IOLoop instance
-    session: StreamSession object
+    session: Session object
     <removed> context: zmq context for creating new connections (?)
     queue: ZMQStream for monitoring the command queue (SUB)
     query: ZMQStream for engine registration and client queries requests (XREP)
@@ -319,7 +336,6 b' class Hub(LoggingFactory):'
     _idcounter=Int(0)
     
     # objects from constructor:
-    loop=Instance(ioloop.IOLoop)
     query=Instance(ZMQStream)
     monitor=Instance(ZMQStream)
     notifier=Instance(ZMQStream)
@@ -438,34 +454,16 b' class Hub(LoggingFactory):'
     # dispatch methods (1 per stream)
     #-----------------------------------------------------------------------------
     
-    # def dispatch_registration_request(self, msg):
-    #     """"""
-    #     self.log.debug("registration::dispatch_register_request(%s)"%msg)
-    #     idents,msg = self.session.feed_identities(msg)
-    #     if not idents:
-    #         self.log.error("Bad Query Message: %s"%msg, exc_info=True)
-    #         return
-    #     try:
-    #         msg = self.session.unpack_message(msg,content=True)
-    #     except:
-    #         self.log.error("registration::got bad registration message: %s"%msg, exc_info=True)
-    #         return
-    #     
-    #     msg_type = msg['msg_type']
-    #     content = msg['content']
-    #     
-    #     handler = self.query_handlers.get(msg_type, None)
-    #     if handler is None:
-    #         self.log.error("registration::got bad registration message: %s"%msg)
-    #     else:
-    #         handler(idents, msg)
     
     def dispatch_monitor_traffic(self, msg):
         """all ME and Task queue messages come through here, as well as
         IOPub traffic."""
         self.log.debug("monitor traffic: %r"%msg[:2])
         switch = msg[0]
-        idents, msg = self.session.feed_identities(msg[1:])
+        try:
+            idents, msg = self.session.feed_identities(msg[1:])
+        except ValueError:
+            idents=[]
         if not idents:
             self.log.error("Bad Monitor Message: %r"%msg)
             return
@@ -478,20 +476,22 b' class Hub(LoggingFactory):'
     
     def dispatch_query(self, msg):
         """Route registration requests and queries from clients."""
-        idents, msg = self.session.feed_identities(msg)
+        try:
+            idents, msg = self.session.feed_identities(msg)
+        except ValueError:
+            idents = []
         if not idents:
             self.log.error("Bad Query Message: %r"%msg)
             return
         client_id = idents[0]
         try:
             msg = self.session.unpack_message(msg, content=True)
-        except:
+        except Exception:
             content = error.wrap_exception()
             self.log.error("Bad Query Message: %r"%msg, exc_info=True)
             self.session.send(self.query, "hub_error", ident=client_id, 
                     content=content)
             return
-        
         # print client_id, header, parent, content
         #switch on message type:
         msg_type = msg['msg_type']
@@ -546,24 +546,22 b' class Hub(LoggingFactory):'
     
     def save_queue_request(self, idents, msg):
         if len(idents) < 2:
-            self.log.error("invalid identity prefix: %s"%idents)
+            self.log.error("invalid identity prefix: %r"%idents)
             return
         queue_id, client_id = idents[:2]
         try:
-            msg = self.session.unpack_message(msg, content=False)
-        except:
-            self.log.error("queue::client %r sent invalid message to %r: %s"%(client_id, queue_id, msg), exc_info=True)
+            msg = self.session.unpack_message(msg)
+        except Exception:
+            self.log.error("queue::client %r sent invalid message to %r: %r"%(client_id, queue_id, msg), exc_info=True)
             return
         
         eid = self.by_ident.get(queue_id, None)
         if eid is None:
             self.log.error("queue::target %r not registered"%queue_id)
-            self.log.debug("queue::    valid are: %s"%(self.by_ident.keys()))
+            self.log.debug("queue::    valid are: %r"%(self.by_ident.keys()))
             return
-            
-        header = msg['header']
-        msg_id = header['msg_id']
         record = init_record(msg)
+        msg_id = record['msg_id']
         record['engine_uuid'] = queue_id
         record['client_uuid'] = client_id
         record['queue'] = 'mux'
@@ -577,30 +575,36 b' class Hub(LoggingFactory):'
                     self.log.warn("conflicting initial state for record: %r:%r <%r> %r"%(msg_id, rvalue, key, evalue))
                 elif evalue and not rvalue:
                     record[key] = evalue
-            self.db.update_record(msg_id, record)
+            try:
+                self.db.update_record(msg_id, record)
+            except Exception:
+                self.log.error("DB Error updating record %r"%msg_id, exc_info=True)
         except KeyError:
-            self.db.add_record(msg_id, record)
+            try:
+                self.db.add_record(msg_id, record)
+            except Exception:
+                self.log.error("DB Error adding record %r"%msg_id, exc_info=True)
+                
         
         self.pending.add(msg_id)
         self.queues[eid].append(msg_id)
     
     def save_queue_result(self, idents, msg):
         if len(idents) < 2:
-            self.log.error("invalid identity prefix: %s"%idents)
+            self.log.error("invalid identity prefix: %r"%idents)
             return
             
         client_id, queue_id = idents[:2]
         try:
-            msg = self.session.unpack_message(msg, content=False)
-        except:
-            self.log.error("queue::engine %r sent invalid message to %r: %s"%(
+            msg = self.session.unpack_message(msg)
+        except Exception:
+            self.log.error("queue::engine %r sent invalid message to %r: %r"%(
                     queue_id,client_id, msg), exc_info=True)
             return
         
         eid = self.by_ident.get(queue_id, None)
         if eid is None:
             self.log.error("queue::unknown engine %r is sending a reply: "%queue_id)
-            # self.log.debug("queue::       %s"%msg[2:])
             return
         
         parent = msg['parent_header']
@@ -615,14 +619,12 b' class Hub(LoggingFactory):'
         elif msg_id not in self.all_completed:
             # it could be a result from a dead engine that died before delivering the
             # result
-            self.log.warn("queue:: unknown msg finished %s"%msg_id)
+            self.log.warn("queue:: unknown msg finished %r"%msg_id)
             return
         # update record anyway, because the unregistration could have been premature
         rheader = msg['header']
-        completed = datetime.strptime(rheader['date'], util.ISO8601)
+        completed = rheader['date']
         started = rheader.get('started', None)
-        if started is not None:
-            started = datetime.strptime(started, util.ISO8601)
         result = {
             'result_header' : rheader,
             'result_content': msg['content'],
@@ -644,9 +646,9 b' class Hub(LoggingFactory):'
         client_id = idents[0]
         
         try:
-            msg = self.session.unpack_message(msg, content=False)
-        except:
-            self.log.error("task::client %r sent invalid task message: %s"%(
+            msg = self.session.unpack_message(msg)
+        except Exception:
+            self.log.error("task::client %r sent invalid task message: %r"%(
                     client_id, msg), exc_info=True)
             return
         record = init_record(msg)
@@ -678,9 +680,15 b' class Hub(LoggingFactory):'
                     self.log.warn("conflicting initial state for record: %r:%r <%r> %r"%(msg_id, rvalue, key, evalue))
                 elif evalue and not rvalue:
                     record[key] = evalue
-            self.db.update_record(msg_id, record)
+            try:
+                self.db.update_record(msg_id, record)
+            except Exception:
+                self.log.error("DB Error updating record %r"%msg_id, exc_info=True)
         except KeyError:
-            self.db.add_record(msg_id, record)
+            try:
+                self.db.add_record(msg_id, record)
+            except Exception:
+                self.log.error("DB Error adding record %r"%msg_id, exc_info=True)
         except Exception:
             self.log.error("DB Error saving task request %r"%msg_id, exc_info=True)
     
@@ -688,11 +696,10 b' class Hub(LoggingFactory):'
         """save the result of a completed task."""
         client_id = idents[0]
         try:
-            msg = self.session.unpack_message(msg, content=False)
-        except:
-            self.log.error("task::invalid task result message send to %r: %s"%(
+            msg = self.session.unpack_message(msg)
+        except Exception:
+            self.log.error("task::invalid task result message send to %r: %r"%(
                     client_id, msg), exc_info=True)
-            raise
             return
         
         parent = msg['parent_header']
@@ -715,10 +722,8 b' class Hub(LoggingFactory):'
                 self.completed[eid].append(msg_id)
                 if msg_id in self.tasks[eid]:
                     self.tasks[eid].remove(msg_id)
-            completed = datetime.strptime(header['date'], util.ISO8601)
+            completed = header['date']
             started = header.get('started', None)
-            if started is not None:
-                started = datetime.strptime(started, util.ISO8601)
             result = {
                 'result_header' : header,
                 'result_content': msg['content'],
@@ -734,12 +739,12 b' class Hub(LoggingFactory):'
                 self.log.error("DB Error saving task request %r"%msg_id, exc_info=True)
             
         else:
-            self.log.debug("task::unknown task %s finished"%msg_id)
+            self.log.debug("task::unknown task %r finished"%msg_id)
     
     def save_task_destination(self, idents, msg):
         try:
             msg = self.session.unpack_message(msg, content=True)
-        except:
+        except Exception:
             self.log.error("task::invalid task tracking message", exc_info=True)
             return
         content = msg['content']
@@ -748,11 +753,11 b' class Hub(LoggingFactory):'
         engine_uuid = content['engine_id']
         eid = self.by_ident[engine_uuid]
         
-        self.log.info("task::task %s arrived on %s"%(msg_id, eid))
+        self.log.info("task::task %r arrived on %r"%(msg_id, eid))
         if msg_id in self.unassigned:
             self.unassigned.remove(msg_id)
         # else:
-        #     self.log.debug("task::task %s not listed as MIA?!"%(msg_id))
+        #     self.log.debug("task::task %r not listed as MIA?!"%(msg_id))
         
         self.tasks[eid].append(msg_id)
         # self.pending[msg_id][1].update(received=datetime.now(),engine=(eid,engine_uuid))
@@ -776,13 +781,13 b' class Hub(LoggingFactory):'
         # print (topics)
         try:
             msg = self.session.unpack_message(msg, content=True)
-        except:
+        except Exception:
             self.log.error("iopub::invalid IOPub message", exc_info=True)
             return
         
         parent = msg['parent_header']
         if not parent:
-            self.log.error("iopub::invalid IOPub message: %s"%msg)
+            self.log.error("iopub::invalid IOPub message: %r"%msg)
             return
         msg_id = parent['msg_id']
         msg_type = msg['msg_type']
@@ -822,7 +827,7 b' class Hub(LoggingFactory):'
         
     def connection_request(self, client_id, msg):
         """Reply with connection addresses for clients."""
-        self.log.info("client::client %s connected"%client_id)
+        self.log.info("client::client %r connected"%client_id)
         content = dict(status='ok')
         content.update(self.client_info)
         jsonable = {}
@@ -894,7 +899,7 b' class Hub(LoggingFactory):'
                 dc.start()
                 self.incoming_registrations[heart] = (eid,queue,reg[0],dc)
         else:
-            self.log.error("registration::registration %i failed: %s"%(eid, content['evalue']))
+            self.log.error("registration::registration %i failed: %r"%(eid, content['evalue']))
         return eid
     
     def unregister_engine(self, ident, msg):
@@ -902,9 +907,9 b' class Hub(LoggingFactory):'
         try:
             eid = msg['content']['id']
         except:
-            self.log.error("registration::bad engine id for unregistration: %s"%ident, exc_info=True)
+            self.log.error("registration::bad engine id for unregistration: %r"%ident, exc_info=True)
             return
-        self.log.info("registration::unregister_engine(%s)"%eid)
+        self.log.info("registration::unregister_engine(%r)"%eid)
         # print (eid)
         uuid = self.keytable[eid]
         content=dict(id=eid, queue=uuid)
@@ -1124,7 +1129,7 b' class Hub(LoggingFactory):'
         elif len(records) < len(msg_ids):
             missing = [ m for m in msg_ids if m not in found_ids ]
             try:
-                raise KeyError("No such msg(s): %s"%missing)
+                raise KeyError("No such msg(s): %r"%missing)
             except KeyError:
                 return finish(error.wrap_exception())
         elif invalid_ids:
@@ -1135,9 +1140,10 b' class Hub(LoggingFactory):'
                 return finish(error.wrap_exception())
 
         # clear the existing records
+        now = datetime.now()
         rec = empty_record()
         map(rec.pop, ['msg_id', 'header', 'content', 'buffers', 'submitted'])
-        rec['resubmitted'] = datetime.now()
+        rec['resubmitted'] = now
         rec['queue'] = 'task'
         rec['client_uuid'] = client_id[0]
         try:
@@ -1151,6 +1157,8 b' class Hub(LoggingFactory):'
             # send the messages
             for rec in records:
                 header = rec['header']
+                # include resubmitted in header to prevent digest collision
+                header['resubmitted'] = now
                 msg = self.session.msg(header['msg_type'])
                 msg['content'] = rec['content']
                 msg['header'] = header
@@ -1246,10 +1254,8 b' class Hub(LoggingFactory):'
         content = msg['content']
         query = content.get('query', {})
         keys = content.get('keys', None)
-        query = util.extract_dates(query)
         buffers = []
         empty = list()
-        
         try:
             records = self.db.find_records(query, keys)
         except Exception as e:

IPython/parallel/controller/mongodb.py

@@ -1,6 +1,11 b''
-"""A TaskRecord backend using mongodb"""
+"""A TaskRecord backend using mongodb
+
+Authors:
+
+* Min RK
+"""
 #-----------------------------------------------------------------------------
-#  Copyright (C) 2010  The IPython Development Team
+#  Copyright (C) 2010-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
@@ -9,7 +14,7 b''
 from pymongo import Connection
 from pymongo.binary import Binary
 
-from IPython.utils.traitlets import Dict, List, CUnicode, CStr, Instance
+from IPython.utils.traitlets import Dict, List, Unicode, Instance
 
 from .dictdb import BaseDB
 
@@ -20,9 +25,20 b' from .dictdb import BaseDB'
 class MongoDB(BaseDB):
     """MongoDB TaskRecord backend."""
     
-    connection_args = List(config=True) # args passed to pymongo.Connection
-    connection_kwargs = Dict(config=True) # kwargs passed to pymongo.Connection
-    database = CUnicode(config=True) # name of the mongodb database
+    connection_args = List(config=True,
+        help="""Positional arguments to be passed to pymongo.Connection.  Only
+        necessary if the default mongodb configuration does not point to your
+        mongod instance.""")
+    connection_kwargs = Dict(config=True,
+        help="""Keyword arguments to be passed to pymongo.Connection.  Only
+        necessary if the default mongodb configuration does not point to your
+        mongod instance."""
+    )
+    database = Unicode(config=True,
+        help="""The MongoDB database name to use for storing tasks for this session. If unspecified,
+        a new database will be created with the Hub's IDENT.  Specifying the database will result
+        in tasks from previous sessions being available via Clients' db_query and
+        get_result methods.""")
 
     _connection = Instance(Connection) # pymongo connection
     

IPython/parallel/controller/scheduler.py

@@ -3,6 +3,10 b''
 The Pure ZMQ scheduler does not allow routing schemes other than LRU,
 nor does it check msg_id DAG dependencies. For those, a slightly slower
 Python Scheduler exists.
+
+Authors:
+
+* Min RK
 """
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2010-2011  The IPython Development Team
@@ -35,7 +39,7 b' from zmq.eventloop import ioloop, zmqstream'
 # local imports
 from IPython.external.decorator import decorator
 from IPython.config.loader import Config
-from IPython.utils.traitlets import Instance, Dict, List, Set, Int
+from IPython.utils.traitlets import Instance, Dict, List, Set, Int, Str, Enum
 
 from IPython.parallel import error
 from IPython.parallel.factory import SessionFactory
@@ -126,10 +130,24 b' class TaskScheduler(SessionFactory):'
     
     """
     
-    hwm = Int(0, config=True) # limit number of outstanding tasks
+    hwm = Int(0, config=True, shortname='hwm',
+        help="""specify the High Water Mark (HWM) for the downstream
+        socket in the Task scheduler. This is the maximum number
+        of allowed outstanding tasks on each engine."""
+    )
+    scheme_name = Enum(('leastload', 'pure', 'lru', 'plainrandom', 'weighted', 'twobin'),
+        'leastload', config=True, shortname='scheme', allow_none=False,
+        help="""select the task scheduler scheme  [default: Python LRU]
+        Options are: 'pure', 'lru', 'plainrandom', 'weighted', 'twobin','leastload'"""
+    )
+    def _scheme_name_changed(self, old, new):
+        self.log.debug("Using scheme %r"%new)
+        self.scheme = globals()[new]
     
     # input arguments:
-    scheme = Instance(FunctionType, default=leastload) # function for determining the destination
+    scheme = Instance(FunctionType) # function for determining the destination
+    def _scheme_default(self):
+        return leastload
     client_stream = Instance(zmqstream.ZMQStream) # client-facing stream
     engine_stream = Instance(zmqstream.ZMQStream) # engine-facing stream
     notifier_stream = Instance(zmqstream.ZMQStream) # hub-facing sub stream
@@ -165,7 +183,7 b' class TaskScheduler(SessionFactory):'
         self.notifier_stream.on_recv(self.dispatch_notification)
         self.auditor = ioloop.PeriodicCallback(self.audit_timeouts, 2e3, self.loop) # 1 Hz
         self.auditor.start()
-        self.log.info("Scheduler started...%r"%self)
+        self.log.info("Scheduler started [%s]"%self.scheme_name)
     
     def resume_receiving(self):
         """Resume accepting jobs."""
@@ -182,17 +200,27 b' class TaskScheduler(SessionFactory):'
     
     def dispatch_notification(self, msg):
         """dispatch register/unregister events."""
-        idents,msg = self.session.feed_identities(msg)
-        msg = self.session.unpack_message(msg)
+        try:
+            idents,msg = self.session.feed_identities(msg)
+        except ValueError:
+            self.log.warn("task::Invalid Message: %r"%msg)
+            return
+        try:
+            msg = self.session.unpack_message(msg)
+        except ValueError:
+            self.log.warn("task::Unauthorized message from: %r"%idents)
+            return
+        
         msg_type = msg['msg_type']
+        
         handler = self._notification_handlers.get(msg_type, None)
         if handler is None:
-            raise Exception("Unhandled message type: %s"%msg_type)
+            self.log.error("Unhandled message type: %r"%msg_type)
         else:
             try:
                 handler(str(msg['content']['queue']))
             except KeyError:
-                self.log.error("task::Invalid notification msg: %s"%msg)
+                self.log.error("task::Invalid notification msg: %r"%msg)
     
     @logged
     def _register_engine(self, uid):
@@ -247,10 +275,8 b' class TaskScheduler(SessionFactory):'
                 continue
 
             raw_msg = lost[msg_id][0]
-
             idents,msg = self.session.feed_identities(raw_msg, copy=False)
-            msg = self.session.unpack_message(msg, copy=False, content=False)
-            parent = msg['header']
+            parent = self.session.unpack(msg[1].bytes)
             idents = [engine, idents[0]]
 
             # build fake error reply
@@ -280,9 +306,10 b' class TaskScheduler(SessionFactory):'
             idents, msg = self.session.feed_identities(raw_msg, copy=False)
             msg = self.session.unpack_message(msg, content=False, copy=False)
         except Exception:
-            self.log.error("task::Invaid task: %s"%raw_msg, exc_info=True)
+            self.log.error("task::Invaid task msg: %r"%raw_msg, exc_info=True)
             return
         
+        
         # send to monitor
         self.mon_stream.send_multipart(['intask']+raw_msg, copy=False)
         
@@ -363,8 +390,7 b' class TaskScheduler(SessionFactory):'
         
         # FIXME: unpacking a message I've already unpacked, but didn't save:
         idents,msg = self.session.feed_identities(raw_msg, copy=False)
-        msg = self.session.unpack_message(msg, copy=False, content=False)
-        header = msg['header']
+        header = self.session.unpack(msg[1].bytes)
         
         try:
             raise why()
@@ -483,7 +509,7 b' class TaskScheduler(SessionFactory):'
             else:
                 self.finish_job(idx)
         except Exception:
-            self.log.error("task::Invaid result: %s"%raw_msg, exc_info=True)
+            self.log.error("task::Invaid result: %r"%raw_msg, exc_info=True)
             return
 
         header = msg['header']
@@ -590,7 +616,8 b' class TaskScheduler(SessionFactory):'
         for msg_id in jobs:
             raw_msg, targets, after, follow, timeout = self.depending[msg_id]
             
-            if after.unreachable(self.all_completed, self.all_failed) or follow.unreachable(self.all_completed, self.all_failed):
+            if after.unreachable(self.all_completed, self.all_failed)\
+                    or follow.unreachable(self.all_completed, self.all_failed):
                 self.fail_unreachable(msg_id)
             
             elif after.check(self.all_completed, self.all_failed): # time deps met, maybe run
@@ -621,9 +648,9 b' class TaskScheduler(SessionFactory):'
     
 
 
-def launch_scheduler(in_addr, out_addr, mon_addr, not_addr, config=None,logname='ZMQ', 
-                            log_addr=None, loglevel=logging.DEBUG, scheme='lru',
-                            identity=b'task'):
+def launch_scheduler(in_addr, out_addr, mon_addr, not_addr, config=None,
+                        logname='root', log_url=None, loglevel=logging.DEBUG,
+                        identity=b'task'):
     from zmq.eventloop import ioloop
     from zmq.eventloop.zmqstream import ZMQStream
     
@@ -646,16 +673,16 b' def launch_scheduler(in_addr, out_addr, mon_addr, not_addr, config=None,logname='
     nots.setsockopt(zmq.SUBSCRIBE, '')
     nots.connect(not_addr)
     
-    scheme = globals().get(scheme, None)
-    # setup logging
-    if log_addr:
-        connect_logger(logname, ctx, log_addr, root="scheduler", loglevel=loglevel)
+    # setup logging. Note that these will not work in-process, because they clobber
+    # existing loggers.
+    if log_url:
+        log = connect_logger(logname, ctx, log_url, root="scheduler", loglevel=loglevel)
     else:
-        local_logger(logname, loglevel)
+        log = local_logger(logname, loglevel)
     
     scheduler = TaskScheduler(client_stream=ins, engine_stream=outs,
                             mon_stream=mons, notifier_stream=nots,
-                            scheme=scheme, loop=loop, logname=logname,
+                            loop=loop, log=log,
                             config=config)
     scheduler.start()
     try:

IPython/parallel/controller/sqlitedb.py

@@ -1,4 +1,9 b''
-"""A TaskRecord backend using sqlite3"""
+"""A TaskRecord backend using sqlite3
+
+Authors:
+
+* Min RK
+"""
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2011  The IPython Development Team
 #
@@ -15,9 +20,9 b' import sqlite3'
 
 from zmq.eventloop import ioloop
 
-from IPython.utils.traitlets import CUnicode, CStr, Instance, List
+from IPython.utils.traitlets import Unicode, Instance, List, Dict
 from .dictdb import BaseDB
-from IPython.parallel.util import ISO8601
+from IPython.utils.jsonutil import date_default, extract_dates, squash_dates
 
 #-----------------------------------------------------------------------------
 # SQLite operators, adapters, and converters
@@ -42,23 +47,14 b' null_operators = {'
 '!=' : "IS NOT NULL",
 }
 
-def _adapt_datetime(dt):
-    return dt.strftime(ISO8601)
-
-def _convert_datetime(ds):
-    if ds is None:
-        return ds
-    else:
-        return datetime.strptime(ds, ISO8601)
-
 def _adapt_dict(d):
-    return json.dumps(d)
+    return json.dumps(d, default=date_default)
 
 def _convert_dict(ds):
     if ds is None:
         return ds
     else:
-        return json.loads(ds)
+        return extract_dates(json.loads(ds))
 
 def _adapt_bufs(bufs):
     # this is *horrible*
@@ -83,11 +79,19 b' def _convert_bufs(bs):'
 class SQLiteDB(BaseDB):
     """SQLite3 TaskRecord backend."""
     
-    filename = CUnicode('tasks.db', config=True)
-    location = CUnicode('', config=True)
-    table = CUnicode("", config=True)
+    filename = Unicode('tasks.db', config=True,
+        help="""The filename of the sqlite task database. [default: 'tasks.db']""")
+    location = Unicode('', config=True,
+        help="""The directory containing the sqlite task database.  The default
+        is to use the cluster_dir location.""")
+    table = Unicode("", config=True,
+        help="""The SQLite Table to use for storing tasks for this session. If unspecified,
+        a new table will be created with the Hub's IDENT.  Specifying the table will result
+        in tasks from previous sessions being available via Clients' db_query and
+        get_result methods.""")
     
     _db = Instance('sqlite3.Connection')
+    # the ordered list of column names
     _keys = List(['msg_id' ,
             'header' ,
             'content',
@@ -108,6 +112,27 b' class SQLiteDB(BaseDB):'
             'stdout',
             'stderr',
         ])
+    # sqlite datatypes for checking that db is current format
+    _types = Dict({'msg_id' : 'text' ,
+            'header' : 'dict text',
+            'content' : 'dict text',
+            'buffers' : 'bufs blob',
+            'submitted' : 'timestamp',
+            'client_uuid' : 'text',
+            'engine_uuid' : 'text',
+            'started' : 'timestamp',
+            'completed' : 'timestamp',
+            'resubmitted' : 'timestamp',
+            'result_header' : 'dict text',
+            'result_content' : 'dict text',
+            'result_buffers' : 'bufs blob',
+            'queue' : 'text',
+            'pyin' : 'text',
+            'pyout' : 'text',
+            'pyerr' : 'text',
+            'stdout' : 'text',
+            'stderr' : 'text',
+        })
     
     def __init__(self, **kwargs):
         super(SQLiteDB, self).__init__(**kwargs)
@@ -115,10 +140,16 b' class SQLiteDB(BaseDB):'
             # use session, and prefix _, since starting with # is illegal
             self.table = '_'+self.session.replace('-','_')
         if not self.location:
-            if hasattr(self.config.Global, 'cluster_dir'):
-                self.location = self.config.Global.cluster_dir
+            # get current profile
+            from IPython.core.application import BaseIPythonApplication
+            if BaseIPythonApplication.initialized():
+                app = BaseIPythonApplication.instance()
+                if app.profile_dir is not None:
+                    self.location = app.profile_dir.location
+                else:
+                    self.location = u'.'
             else:
-                self.location = '.'
+                self.location = u'.'
         self._init_db()
         
         # register db commit as 2s periodic callback
@@ -136,11 +167,36 b' class SQLiteDB(BaseDB):'
             d[key] = None
         return d
     
+    def _check_table(self):
+        """Ensure that an incorrect table doesn't exist
+        
+        If a bad (old) table does exist, return False
+        """
+        cursor = self._db.execute("PRAGMA table_info(%s)"%self.table)
+        lines = cursor.fetchall()
+        if not lines:
+            # table does not exist
+            return True
+        types = {}
+        keys = []
+        for line in lines:
+            keys.append(line[1])
+            types[line[1]] = line[2]
+        if self._keys != keys:
+            # key mismatch
+            self.log.warn('keys mismatch')
+            return False
+        for key in self._keys:
+            if types[key] != self._types[key]:
+                self.log.warn(
+                    'type mismatch: %s: %s != %s'%(key,types[key],self._types[key])
+                )
+                return False
+        return True
+        
     def _init_db(self):
         """Connect to the database and get new session number."""
         # register adapters
-        sqlite3.register_adapter(datetime, _adapt_datetime)
-        sqlite3.register_converter('datetime', _convert_datetime)
         sqlite3.register_adapter(dict, _adapt_dict)
         sqlite3.register_converter('dict', _convert_dict)
         sqlite3.register_adapter(list, _adapt_bufs)
@@ -151,18 +207,27 b' class SQLiteDB(BaseDB):'
             # isolation_level = None)#,
              cached_statements=64)
         # print dir(self._db)
+        first_table = self.table
+        i=0
+        while not self._check_table():
+            i+=1
+            self.table = first_table+'_%i'%i
+            self.log.warn(
+                "Table %s exists and doesn't match db format, trying %s"%
+                (first_table,self.table)
+            )
         
         self._db.execute("""CREATE TABLE IF NOT EXISTS %s 
                 (msg_id text PRIMARY KEY,
                 header dict text,
                 content dict text,
                 buffers bufs blob,
-                submitted datetime text,
+                submitted timestamp,
                 client_uuid text,
                 engine_uuid text,
-                started datetime text,
-                completed datetime text,
-                resubmitted datetime text,
+                started timestamp,
+                completed timestamp,
+                resubmitted timestamp,
                 result_header dict text,
                 result_content dict text,
                 result_buffers bufs blob,

IPython/parallel/engine/engine.py

@@ -2,6 +2,10 b''
 """A simple engine that talks to a controller over 0MQ.
 it handles registration, etc. and launches a kernel
 connected to the Controller's Schedulers.
+
+Authors:
+
+* Min RK
 """
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2010-2011  The IPython Development Team
@@ -19,27 +23,37 b' import zmq'
 from zmq.eventloop import ioloop, zmqstream
 
 # internal
-from IPython.utils.traitlets import Instance, Str, Dict, Int, Type, CFloat
+from IPython.utils.traitlets import Instance, Dict, Int, Type, CFloat, Unicode
 # from IPython.utils.localinterfaces import LOCALHOST 
 
 from IPython.parallel.controller.heartmonitor import Heart
 from IPython.parallel.factory import RegistrationFactory
-from IPython.parallel.streamsession import Message
 from IPython.parallel.util import disambiguate_url
 
+from IPython.zmq.session import Message
+
 from .streamkernel import Kernel
 
 class EngineFactory(RegistrationFactory):
     """IPython engine"""
     
     # configurables:
-    user_ns=Dict(config=True)
-    out_stream_factory=Type('IPython.zmq.iostream.OutStream', config=True)
-    display_hook_factory=Type('IPython.zmq.displayhook.DisplayHook', config=True)
-    location=Str(config=True)
-    timeout=CFloat(2,config=True)
+    out_stream_factory=Type('IPython.zmq.iostream.OutStream', config=True,
+        help="""The OutStream for handling stdout/err.
+        Typically 'IPython.zmq.iostream.OutStream'""")
+    display_hook_factory=Type('IPython.zmq.displayhook.DisplayHook', config=True,
+        help="""The class for handling displayhook.
+        Typically 'IPython.zmq.displayhook.DisplayHook'""")
+    location=Unicode(config=True,
+        help="""The location (an IP address) of the controller.  This is
+        used for disambiguating URLs, to determine whether
+        loopback should be used to connect or the public address.""")
+    timeout=CFloat(2,config=True,
+        help="""The time (in seconds) to wait for the Controller to respond
+        to registration requests before giving up.""")
     
     # not configurable:
+    user_ns=Dict()
     id=Int(allow_none=True)
     registrar=Instance('zmq.eventloop.zmqstream.ZMQStream')
     kernel=Instance(Kernel)
@@ -47,6 +61,7 b' class EngineFactory(RegistrationFactory):'
     
     def __init__(self, **kwargs):
         super(EngineFactory, self).__init__(**kwargs)
+        self.ident = self.session.session
         ctx = self.context
         
         reg = ctx.socket(zmq.XREQ)
@@ -127,11 +142,10 b' class EngineFactory(RegistrationFactory):'
             
             self.kernel = Kernel(config=self.config, int_id=self.id, ident=self.ident, session=self.session, 
                     control_stream=control_stream, shell_streams=shell_streams, iopub_stream=iopub_stream, 
-                    loop=loop, user_ns = self.user_ns, logname=self.log.name)
+                    loop=loop, user_ns = self.user_ns, log=self.log)
             self.kernel.start()
             hb_addrs = [ disambiguate_url(addr, self.location) for addr in hb_addrs ]
             heart = Heart(*map(str, hb_addrs), heart_id=identity)
-            # ioloop.DelayedCallback(heart.start, 1000, self.loop).start()
             heart.start()
             
             
@@ -143,7 +157,7 b' class EngineFactory(RegistrationFactory):'
     
     
     def abort(self):
-        self.log.fatal("Registration timed out")
+        self.log.fatal("Registration timed out after %.1f seconds"%self.timeout)
         self.session.send(self.registrar, "unregistration_request", content=dict(id=self.id))
         time.sleep(1)
         sys.exit(255)

IPython/parallel/engine/kernelstarter.py

@@ -1,4 +1,9 b''
-"""KernelStarter class that intercepts Control Queue messages, and handles process management."""
+"""KernelStarter class that intercepts Control Queue messages, and handles process management.
+
+Authors:
+
+* Min RK
+"""
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2010-2011  The IPython Development Team
 #
@@ -8,7 +13,7 b''
 
 from zmq.eventloop import ioloop
 
-from IPython.parallel.streamsession import StreamSession
+from IPython.zmq.session import Session
 
 class KernelStarter(object):
     """Object for resetting/killing the Kernel."""
@@ -213,7 +218,7 b' def make_starter(up_addr, down_addr, *args, **kwargs):'
     """entry point function for launching a kernelstarter in a subprocess"""
     loop = ioloop.IOLoop.instance()
     ctx = zmq.Context()
-    session = StreamSession()
+    session = Session()
     upstream = zmqstream.ZMQStream(ctx.socket(zmq.XREQ),loop)
     upstream.connect(up_addr)
     downstream = zmqstream.ZMQStream(ctx.socket(zmq.XREQ),loop)

IPython/parallel/engine/streamkernel.py

@@ -1,6 +1,13 b''
 #!/usr/bin/env python
 """
 Kernel adapted from kernel.py to use ZMQ Streams
+
+Authors:
+
+* Min RK
+* Brian Granger
+* Fernando Perez
+* Evan Patterson
 """
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2010-2011  The IPython Development Team
@@ -28,12 +35,12 b' import zmq'
 from zmq.eventloop import ioloop, zmqstream
 
 # Local imports.
-from IPython.utils.traitlets import Instance, List, Int, Dict, Set, Str
+from IPython.utils.traitlets import Instance, List, Int, Dict, Set, Unicode
 from IPython.zmq.completer import KernelCompleter
 
 from IPython.parallel.error import wrap_exception
 from IPython.parallel.factory import SessionFactory
-from IPython.parallel.util import serialize_object, unpack_apply_message, ISO8601
+from IPython.parallel.util import serialize_object, unpack_apply_message
 
 def printer(*args):
     pprint(args, stream=sys.__stdout__)
@@ -42,7 +49,7 b' def printer(*args):'
 class _Passer(zmqstream.ZMQStream):
     """Empty class that implements `send()` that does nothing.
     
-    Subclass ZMQStream for StreamSession typechecking
+    Subclass ZMQStream for Session typechecking
     
     """
     def __init__(self, *args, **kwargs):
@@ -64,9 +71,11 b' class Kernel(SessionFactory):'
     #---------------------------------------------------------------------------
     
     # kwargs:
-    int_id = Int(-1, config=True)
-    user_ns = Dict(config=True)
-    exec_lines = List(config=True)
+    exec_lines = List(Unicode, config=True,
+        help="List of lines to execute")
+
+    int_id = Int(-1)
+    user_ns = Dict(config=True,  help="""Set the user's namespace of the Kernel""")
     
     control_stream = Instance(zmqstream.ZMQStream)
     task_stream = Instance(zmqstream.ZMQStream)
@@ -129,21 +138,10 b' class Kernel(SessionFactory):'
     
     def abort_queue(self, stream):
         while True:
-            try:
-                msg = self.session.recv(stream, zmq.NOBLOCK,content=True)
-            except zmq.ZMQError as e:
-                if e.errno == zmq.EAGAIN:
-                    break
-                else:
-                    return
-            else:
-                if msg is None:
-                    return
-                else:
-                    idents,msg = msg
+            idents,msg = self.session.recv(stream, zmq.NOBLOCK, content=True)
+            if msg is None:
+                return
                 
-                # assert self.reply_socketly_socket.rcvmore(), "Unexpected missing message part."
-                # msg = self.reply_socket.recv_json()
             self.log.info("Aborting:")
             self.log.info(str(msg))
             msg_type = msg['msg_type']
@@ -250,7 +248,7 b' class Kernel(SessionFactory):'
             return
         self.session.send(self.iopub_stream, u'pyin', {u'code':code},parent=parent,
                             ident='%s.pyin'%self.prefix)
-        started = datetime.now().strftime(ISO8601)
+        started = datetime.now()
         try:
             comp_code = self.compiler(code, '<zmq-kernel>')
             # allow for not overriding displayhook
@@ -300,7 +298,7 b' class Kernel(SessionFactory):'
         # self.iopub_stream.send(pyin_msg)
         # self.session.send(self.iopub_stream, u'pyin', {u'code':code},parent=parent)
         sub = {'dependencies_met' : True, 'engine' : self.ident,
-                'started': datetime.now().strftime(ISO8601)}
+                'started': datetime.now()}
         try:
             # allow for not overriding displayhook
             if hasattr(sys.displayhook, 'set_parent'):

IPython/parallel/error.py

@@ -1,6 +1,12 b''
 # encoding: utf-8
 
-"""Classes and functions for kernel related errors and exceptions."""
+"""Classes and functions for kernel related errors and exceptions.
+
+Authors:
+
+* Brian Granger
+* Min RK
+"""
 from __future__ import print_function
 
 import sys
@@ -12,7 +18,7 b' __docformat__ = "restructuredtext en"'
 __test__ = {}
 
 #-------------------------------------------------------------------------------
-#  Copyright (C) 2008  The IPython Development Team
+#  Copyright (C) 2008-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.

IPython/parallel/factory.py

@@ -1,7 +1,12 b''
-"""Base config factories."""
+"""Base config factories.
+
+Authors:
+
+* Min RK
+"""
 
 #-----------------------------------------------------------------------------
-#  Copyright (C) 2008-2009  The IPython Development Team
+#  Copyright (C) 2010-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
@@ -14,75 +19,38 b''
 
 import logging
 import os
-import uuid
 
+import zmq
 from zmq.eventloop.ioloop import IOLoop
 
 from IPython.config.configurable import Configurable
-from IPython.utils.importstring import import_item
-from IPython.utils.traitlets import Str,Int,Instance, CUnicode, CStr
+from IPython.utils.traitlets import Int, Instance, Unicode
 
-import IPython.parallel.streamsession as ss
 from IPython.parallel.util import select_random_ports
+from IPython.zmq.session import Session, SessionFactory
 
 #-----------------------------------------------------------------------------
 # Classes
 #-----------------------------------------------------------------------------
-class LoggingFactory(Configurable):
-    """A most basic class, that has a `log` (type:`Logger`) attribute, set via a `logname` Trait."""
-    log = Instance('logging.Logger', ('ZMQ', logging.WARN))
-    logname = CUnicode('ZMQ')
-    def _logname_changed(self, name, old, new):
-        self.log = logging.getLogger(new)
-    
 
-class SessionFactory(LoggingFactory):
-    """The Base factory from which every factory in IPython.parallel inherits"""
-    
-    packer = Str('',config=True)
-    unpacker = Str('',config=True)
-    ident = CStr('',config=True)
-    def _ident_default(self):
-        return str(uuid.uuid4())
-    username = CUnicode(os.environ.get('USER','username'),config=True)
-    exec_key = CUnicode('',config=True)
-    # not configurable:
-    context = Instance('zmq.Context', (), {})
-    session = Instance('IPython.parallel.streamsession.StreamSession')
-    loop = Instance('zmq.eventloop.ioloop.IOLoop', allow_none=False)
-    def _loop_default(self):
-        return IOLoop.instance()
-    
-    
-    def __init__(self, **kwargs):
-        super(SessionFactory, self).__init__(**kwargs)
-        exec_key = self.exec_key or None
-        # set the packers:
-        if not self.packer:
-            packer_f = unpacker_f = None
-        elif self.packer.lower() == 'json':
-            packer_f = ss.json_packer
-            unpacker_f = ss.json_unpacker
-        elif self.packer.lower() == 'pickle':
-            packer_f = ss.pickle_packer
-            unpacker_f = ss.pickle_unpacker
-        else:
-            packer_f = import_item(self.packer)
-            unpacker_f = import_item(self.unpacker)
-        
-        # construct the session
-        self.session = ss.StreamSession(self.username, self.ident, packer=packer_f, unpacker=unpacker_f, key=exec_key)
-    
 
 class RegistrationFactory(SessionFactory):
     """The Base Configurable for objects that involve registration."""
     
-    url = Str('', config=True) # url takes precedence over ip,regport,transport
-    transport = Str('tcp', config=True)
-    ip = Str('127.0.0.1', config=True)
-    regport = Instance(int, config=True)
+    url = Unicode('', config=True,
+        help="""The 0MQ url used for registration. This sets transport, ip, and port
+        in one variable. For example: url='tcp://127.0.0.1:12345' or
+        url='epgm://*:90210'""") # url takes precedence over ip,regport,transport
+    transport = Unicode('tcp', config=True,
+        help="""The 0MQ transport for communications.  This will likely be
+        the default of 'tcp', but other values include 'ipc', 'epgm', 'inproc'.""")
+    ip = Unicode('127.0.0.1', config=True,
+        help="""The IP address for registration.  This is generally either
+        '127.0.0.1' for loopback only or '*' for all interfaces.
+        [default: '127.0.0.1']""")
+    regport = Int(config=True,
+        help="""The port on which the Hub listens for registration.""")
     def _regport_default(self):
-        # return 10101
         return select_random_ports(1)[0]
     
     def __init__(self, **kwargs):
@@ -107,46 +75,3 b' class RegistrationFactory(SessionFactory):'
             self.ip = iface[0]
             if iface[1]:
                 self.regport = int(iface[1])
-
-#-----------------------------------------------------------------------------
-# argparse argument extenders
-#-----------------------------------------------------------------------------
-
-
-def add_session_arguments(parser):
-    paa = parser.add_argument
-    paa('--ident',
-        type=str, dest='SessionFactory.ident', 
-        help='set the ZMQ and session identity [default: random uuid]',
-        metavar='identity')
-    # paa('--execkey',
-    #     type=str, dest='SessionFactory.exec_key', 
-    #     help='path to a file containing an execution key.',
-    #     metavar='execkey')
-    paa('--packer',
-        type=str, dest='SessionFactory.packer', 
-        help='method to serialize messages: {json,pickle} [default: json]',
-        metavar='packer')
-    paa('--unpacker',
-        type=str, dest='SessionFactory.unpacker', 
-        help='inverse function of `packer`.  Only necessary when using something other than json|pickle',
-        metavar='packer')
-
-def add_registration_arguments(parser):
-    paa = parser.add_argument
-    paa('--ip',
-        type=str, dest='RegistrationFactory.ip',
-        help="The IP used for registration [default: localhost]",
-        metavar='ip')
-    paa('--transport',
-        type=str, dest='RegistrationFactory.transport',
-        help="The ZeroMQ transport used for registration [default: tcp]",
-        metavar='transport')
-    paa('--url',
-        type=str, dest='RegistrationFactory.url', 
-        help='set transport,ip,regport in one go, e.g. tcp://127.0.0.1:10101',
-        metavar='url')
-    paa('--regport',
-        type=int, dest='RegistrationFactory.regport',
-        help="The port used for registration [default: 10101]",
-        metavar='ip')

IPython/parallel/tests/__init__.py

@@ -48,10 +48,10 b' class TestProcessLauncher(LocalProcessLauncher):'
 def setup():
     cp = TestProcessLauncher()
     cp.cmd_and_args = ipcontroller_cmd_argv + \
-                ['--profile', 'iptest', '--log-level', '99', '-r']
+                ['profile=iptest', 'log_level=50', '--reuse']
     cp.start()
     launchers.append(cp)
-    cluster_dir = os.path.join(get_ipython_dir(), 'cluster_iptest')
+    cluster_dir = os.path.join(get_ipython_dir(), 'profile_iptest')
     engine_json = os.path.join(cluster_dir, 'security', 'ipcontroller-engine.json')
     client_json = os.path.join(cluster_dir, 'security', 'ipcontroller-client.json')
     tic = time.time()
@@ -70,7 +70,7 b" def add_engines(n=1, profile='iptest'):"
     eps = []
     for i in range(n):
         ep = TestProcessLauncher()
-        ep.cmd_and_args = ipengine_cmd_argv + ['--profile', profile, '--log-level', '99']
+        ep.cmd_and_args = ipengine_cmd_argv + ['profile=%s'%profile, 'log_level=50']
         ep.start()
         launchers.append(ep)
         eps.append(ep)

IPython/parallel/tests/clienttest.py

@@ -1,4 +1,9 b''
-"""base class for parallel client tests"""
+"""base class for parallel client tests
+
+Authors:
+
+* Min RK
+"""
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2011  The IPython Development Team

IPython/parallel/tests/test_asyncresult.py

@@ -1,4 +1,9 b''
-"""Tests for asyncresult.py"""
+"""Tests for asyncresult.py
+
+Authors:
+
+* Min RK
+"""
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2011  The IPython Development Team

IPython/parallel/tests/test_client.py

@@ -1,4 +1,9 b''
-"""Tests for parallel client.py"""
+"""Tests for parallel client.py
+
+Authors:
+
+* Min RK
+"""
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2011  The IPython Development Team

IPython/parallel/tests/test_db.py

@@ -1,4 +1,9 b''
-"""Tests for db backends"""
+"""Tests for db backends
+
+Authors:
+
+* Min RK
+"""
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2011  The IPython Development Team
@@ -20,18 +25,21 b' from unittest import TestCase'
 
 from nose import SkipTest
 
-from IPython.parallel import error, streamsession as ss
+from IPython.parallel import error
 from IPython.parallel.controller.dictdb import DictDB
 from IPython.parallel.controller.sqlitedb import SQLiteDB
 from IPython.parallel.controller.hub import init_record, empty_record
 
+from IPython.zmq.session import Session
+
+
 #-------------------------------------------------------------------------------
 # TestCases
 #-------------------------------------------------------------------------------
 
 class TestDictBackend(TestCase):
     def setUp(self):
-        self.session = ss.StreamSession()
+        self.session = Session()
         self.db = self.create_db()
         self.load_records(16)
     

IPython/parallel/tests/test_dependency.py

@@ -1,4 +1,9 b''
-"""Tests for dependency.py"""
+"""Tests for dependency.py
+
+Authors:
+
+* Min RK
+"""
 
 __docformat__ = "restructuredtext en"
 

IPython/parallel/tests/test_lbview.py

@@ -1,5 +1,10 b''
-"""test LoadBalancedView objects"""
 # -*- coding: utf-8 -*-
+"""test LoadBalancedView objects
+
+Authors:
+
+* Min RK
+"""
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2011  The IPython Development Team
 #

IPython/parallel/tests/test_mongodb.py

@@ -1,4 +1,9 b''
-"""Tests for mongodb backend"""
+"""Tests for mongodb backend
+
+Authors:
+
+* Min RK
+"""
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2011  The IPython Development Team

IPython/parallel/tests/test_newserialized.py

@@ -1,4 +1,9 b''
-"""test serialization with newserialized"""
+"""test serialization with newserialized
+
+Authors:
+
+* Min RK
+"""
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2011  The IPython Development Team

IPython/parallel/tests/test_view.py

@@ -1,5 +1,10 b''
-"""test View objects"""
 # -*- coding: utf-8 -*-
+"""test View objects
+
+Authors:
+
+* Min RK
+"""
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2011  The IPython Development Team
 #
@@ -315,6 +320,7 b' class TestView(ClusterTestCase):'
         sys.stdout = savestdout
         sio.read()
         self.assertTrue('[stdout:%i]'%v.targets in sio.buf)
+        self.assertTrue(sio.buf.rstrip().endswith('10'))
         self.assertRaisesRemote(ZeroDivisionError, ip.magic_px, '1/0')
 
     def test_magic_px_nonblocking(self):

IPython/parallel/util.py

@@ -1,4 +1,9 b''
-"""some generic utilities for dealing with classes, urls, and serialization"""
+"""some generic utilities for dealing with classes, urls, and serialization
+
+Authors:
+
+* Min RK
+"""
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2010-2011  The IPython Development Team
 #
@@ -17,7 +22,6 b' import re'
 import stat
 import socket
 import sys
-from datetime import datetime
 from signal import signal, SIGINT, SIGABRT, SIGTERM
 try:
     from signal import SIGKILL
@@ -40,10 +44,6 b' from IPython.utils.pickleutil import can, uncan, canSequence, uncanSequence'
 from IPython.utils.newserialized import serialize, unserialize
 from IPython.zmq.log import EnginePUBHandler
 
-# globals
-ISO8601="%Y-%m-%dT%H:%M:%S.%f"
-ISO8601_RE=re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+$")
-
 #-----------------------------------------------------------------------------
 # Classes
 #-----------------------------------------------------------------------------
@@ -101,18 +101,6 b' class ReverseDict(dict):'
 # Functions
 #-----------------------------------------------------------------------------
 
-def extract_dates(obj):
-    """extract ISO8601 dates from unpacked JSON"""
-    if isinstance(obj, dict):
-        for k,v in obj.iteritems():
-            obj[k] = extract_dates(v)
-    elif isinstance(obj, list):
-        obj = [ extract_dates(o) for o in obj ]
-    elif isinstance(obj, basestring):
-        if ISO8601_RE.match(obj):
-            obj = datetime.strptime(obj, ISO8601)
-    return obj
-
 def validate_url(url):
     """validate a url for zeromq"""
     if not isinstance(url, basestring):
@@ -194,29 +182,6 b' def disambiguate_url(url, location=None):'
     
     return "%s://%s:%s"%(proto,ip,port)
 
-
-def rekey(dikt):
-    """Rekey a dict that has been forced to use str keys where there should be
-    ints by json.  This belongs in the jsonutil added by fperez."""
-    for k in dikt.iterkeys():
-        if isinstance(k, str):
-            ik=fk=None
-            try:
-                ik = int(k)
-            except ValueError:
-                try:
-                    fk = float(k)
-                except ValueError:
-                    continue
-            if ik is not None:
-                nk = ik
-            else:
-                nk = fk
-            if nk in dikt:
-                raise KeyError("already have key %r"%nk)
-            dikt[nk] = dikt.pop(k)
-    return dikt
-
 def serialize_object(obj, threshold=64e-6):
     """Serialize an object into a list of sendable buffers.
     
@@ -469,6 +434,7 b' def connect_engine_logger(context, iface, engine, loglevel=logging.DEBUG):'
     handler.setLevel(loglevel)
     logger.addHandler(handler)
     logger.setLevel(loglevel)
+    return logger
 
 def local_logger(logname, loglevel=logging.DEBUG):
     loglevel = integer_loglevel(loglevel)
@@ -480,4 +446,5 b' def local_logger(logname, loglevel=logging.DEBUG):'
     handler.setLevel(loglevel)
     logger.addHandler(handler)
     logger.setLevel(loglevel)
+    return logger
 

IPython/testing/tools.py

@@ -158,8 +158,8 b' def default_argv():'
 
     return ['--quick', # so no config file is loaded
             # Other defaults to minimize side effects on stdout
-            '--colors=NoColor', '--no-term-title','--no-banner',
-            '--autocall=0']
+            'colors=NoColor', '--no-term-title','--no-banner',
+            'autocall=0']
 
 
 def default_config():
@@ -197,7 +197,10 b' def ipexec(fname, options=None):'
     
     # For these subprocess calls, eliminate all prompt printing so we only see
     # output from script execution
-    prompt_opts = ['--prompt-in1=""', '--prompt-in2=""', '--prompt-out=""']
+    prompt_opts = [ 'InteractiveShell.prompt_in1=""', 
+                    'InteractiveShell.prompt_in2=""', 
+                    'InteractiveShell.prompt_out=""'
+    ]
     cmdargs = ' '.join(default_argv() + prompt_opts + options)
     
     _ip = get_ipython()

IPython/utils/jsonutil.py

@@ -11,12 +11,79 b''
 # Imports
 #-----------------------------------------------------------------------------
 # stdlib
+import re
 import types
+from datetime import datetime
+
+#-----------------------------------------------------------------------------
+# Globals and constants
+#-----------------------------------------------------------------------------
+
+# timestamp formats
+ISO8601="%Y-%m-%dT%H:%M:%S.%f"
+ISO8601_PAT=re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+$")
 
 #-----------------------------------------------------------------------------
 # Classes and functions
 #-----------------------------------------------------------------------------
 
+def rekey(dikt):
+    """Rekey a dict that has been forced to use str keys where there should be
+    ints by json."""
+    for k in dikt.iterkeys():
+        if isinstance(k, basestring):
+            ik=fk=None
+            try:
+                ik = int(k)
+            except ValueError:
+                try:
+                    fk = float(k)
+                except ValueError:
+                    continue
+            if ik is not None:
+                nk = ik
+            else:
+                nk = fk
+            if nk in dikt:
+                raise KeyError("already have key %r"%nk)
+            dikt[nk] = dikt.pop(k)
+    return dikt
+
+
+def extract_dates(obj):
+    """extract ISO8601 dates from unpacked JSON"""
+    if isinstance(obj, dict):
+        obj = dict(obj) # don't clobber
+        for k,v in obj.iteritems():
+            obj[k] = extract_dates(v)
+    elif isinstance(obj, (list, tuple)):
+        obj = [ extract_dates(o) for o in obj ]
+    elif isinstance(obj, basestring):
+        if ISO8601_PAT.match(obj):
+            obj = datetime.strptime(obj, ISO8601)
+    return obj
+
+def squash_dates(obj):
+    """squash datetime objects into ISO8601 strings"""
+    if isinstance(obj, dict):
+        obj = dict(obj) # don't clobber
+        for k,v in obj.iteritems():
+            obj[k] = squash_dates(v)
+    elif isinstance(obj, (list, tuple)):
+        obj = [ squash_dates(o) for o in obj ]
+    elif isinstance(obj, datetime):
+        obj = obj.strftime(ISO8601)
+    return obj
+    
+def date_default(obj):
+    """default function for packing datetime objects in JSON."""
+    if isinstance(obj, datetime):
+        return obj.strftime(ISO8601)
+    else:
+        raise TypeError("%r is not JSON serializable"%obj)
+
+
+
 def json_clean(obj):
     """Clean an object to ensure it's safe to encode in JSON.
     

IPython/utils/path.py

@@ -413,13 +413,20 b' def check_for_old_config(ipython_dir=None):'
     if ipython_dir is None:
         ipython_dir = get_ipython_dir()
 
-    old_configs = ['ipy_user_conf.py', 'ipythonrc']
+    old_configs = ['ipy_user_conf.py', 'ipythonrc', 'ipython_config.py']
+    warned = False
     for cfg in old_configs:
         f = os.path.join(ipython_dir, cfg)
         if os.path.exists(f):
-            warn.warn("""Found old IPython config file %r.
-    The IPython configuration system has changed as of 0.11, and this file will be ignored.
-    See http://ipython.github.com/ipython-doc/dev/config for details on the new config system.
-    The current default config file is 'ipython_config.py', where you can suppress these
-    warnings with `Global.ignore_old_config = True`."""%f)
+            warned = True
+            warn.warn("Found old IPython config file %r"%f)
+    
+    if warned:
+        warn.warn("""
+    The IPython configuration system has changed as of 0.11, and these files
+    will be ignored. See http://ipython.github.com/ipython-doc/dev/config for
+    details on the new config system. To start configuring IPython, do 
+    `ipython profile create`, and edit `ipython_config.py` in
+    <ipython_dir>/profile_default.""")
+        
 

IPython/utils/text.py

@@ -19,6 +19,8 b' import __main__'
 import os
 import re
 import shutil
+import textwrap
+from string import Formatter
 
 from IPython.external.path import path
 
@@ -391,15 +393,39 b' def igrep(pat,list):'
     return grep(pat,list,case=0)
 
 
-def indent(str,nspaces=4,ntabs=0):
+def indent(instr,nspaces=4, ntabs=0, flatten=False):
     """Indent a string a given number of spaces or tabstops.
 
     indent(str,nspaces=4,ntabs=0) -> indent str by ntabs+nspaces.
+    
+    Parameters
+    ----------
+    
+    instr : basestring
+        The string to be indented.
+    nspaces : int (default: 4)
+        The number of spaces to be indented.
+    ntabs : int (default: 0)
+        The number of tabs to be indented.
+    flatten : bool (default: False)
+        Whether to scrub existing indentation.  If True, all lines will be
+        aligned to the same indentation.  If False, existing indentation will
+        be strictly increased.
+    
+    Returns
+    -------
+    
+    str|unicode : string indented by ntabs and nspaces.
+    
     """
-    if str is None:
+    if instr is None:
         return
     ind = '\t'*ntabs+' '*nspaces
-    outstr = '%s%s' % (ind,str.replace(os.linesep,os.linesep+ind))
+    if flatten:
+        pat = re.compile(r'^\s*', re.MULTILINE)
+    else:
+        pat = re.compile(r'^', re.MULTILINE)
+    outstr = re.sub(pat, ind, instr)
     if outstr.endswith(os.linesep+ind):
         return outstr[:-len(ind)]
     else:
@@ -495,3 +521,93 b' def format_screen(strng):'
     strng = par_re.sub('',strng)
     return strng
 
+def dedent(text):
+    """Equivalent of textwrap.dedent that ignores unindented first line.
+    
+    This means it will still dedent strings like:
+    '''foo
+    is a bar
+    '''
+    
+    For use in wrap_paragraphs.
+    """
+    
+    if text.startswith('\n'):
+        # text starts with blank line, don't ignore the first line
+        return textwrap.dedent(text)
+    
+    # split first line
+    splits = text.split('\n',1)
+    if len(splits) == 1:
+        # only one line
+        return textwrap.dedent(text)
+    
+    first, rest = splits
+    # dedent everything but the first line
+    rest = textwrap.dedent(rest)
+    return '\n'.join([first, rest])
+
+def wrap_paragraphs(text, ncols=80):
+    """Wrap multiple paragraphs to fit a specified width.
+    
+    This is equivalent to textwrap.wrap, but with support for multiple
+    paragraphs, as separated by empty lines.
+    
+    Returns
+    -------
+    
+    list of complete paragraphs, wrapped to fill `ncols` columns.
+    """
+    paragraph_re = re.compile(r'\n(\s*\n)+', re.MULTILINE)
+    text = dedent(text).strip()
+    paragraphs = paragraph_re.split(text)[::2] # every other entry is space
+    out_ps = []
+    indent_re = re.compile(r'\n\s+', re.MULTILINE)
+    for p in paragraphs:
+        # presume indentation that survives dedent is meaningful formatting, 
+        # so don't fill unless text is flush.
+        if indent_re.search(p) is None:
+            # wrap paragraph
+            p = textwrap.fill(p, ncols)
+        out_ps.append(p)
+    return out_ps
+    
+
+
+class EvalFormatter(Formatter):
+    """A String Formatter that allows evaluation of simple expressions.
+    
+    Any time a format key is not found in the kwargs,
+    it will be tried as an expression in the kwargs namespace.
+    
+    This is to be used in templating cases, such as the parallel batch
+    script templates, where simple arithmetic on arguments is useful.
+    
+    Examples
+    --------
+    
+    In [1]: f = EvalFormatter()
+    In [2]: f.format('{n/4}', n=8)
+    Out[2]: '2'
+    
+    In [3]: f.format('{range(3)}')
+    Out[3]: '[0, 1, 2]'
+
+    In [4]: f.format('{3*2}')
+    Out[4]: '6'
+    """
+    
+    def get_value(self, key, args, kwargs):
+        if isinstance(key, (int, long)):
+            return args[key]
+        elif key in kwargs:
+            return kwargs[key]
+        else:
+            # evaluate the expression using kwargs as namespace
+            try:
+                return eval(key, kwargs)
+            except Exception:
+                # classify all bad expressions as key errors
+                raise KeyError(key)
+
+

IPython/zmq/__init__.py

@@ -6,10 +6,10 b''
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
-# Verify zmq version dependency >= 2.0.10-1
+# Verify zmq version dependency >= 2.1.4
 #-----------------------------------------------------------------------------
 
-minimum_pyzmq_version = "2.0.10-1"
+minimum_pyzmq_version = "2.1.4"
 
 try:
     import zmq

IPython/zmq/blockingkernelmanager.py

@@ -21,8 +21,8 b' from Queue import Queue, Empty'
 from IPython.utils import io
 from IPython.utils.traitlets import Type
 
-from .kernelmanager import (KernelManager, SubSocketChannel, 
-                           XReqSocketChannel, RepSocketChannel, HBSocketChannel)
+from .kernelmanager import (KernelManager, SubSocketChannel, HBSocketChannel,
+                           ShellSocketChannel, StdInSocketChannel)
 
 #-----------------------------------------------------------------------------
 # Functions and classes
@@ -61,15 +61,15 b' class BlockingSubSocketChannel(SubSocketChannel):'
         return msgs
 
 
-class BlockingXReqSocketChannel(XReqSocketChannel):
+class BlockingShellSocketChannel(ShellSocketChannel):
 
     def __init__(self, context, session, address=None):
-        super(BlockingXReqSocketChannel, self).__init__(context, session,
+        super(BlockingShellSocketChannel, self).__init__(context, session,
                                                         address)
         self._in_queue = Queue()
 
     def call_handlers(self, msg):
-        #io.rprint('[[XReq]]', msg) # dbg
+        #io.rprint('[[Shell]]', msg) # dbg
         self._in_queue.put(msg)
 
     def msg_ready(self):
@@ -94,7 +94,7 b' class BlockingXReqSocketChannel(XReqSocketChannel):'
         return msgs
     
 
-class BlockingRepSocketChannel(RepSocketChannel):
+class BlockingStdInSocketChannel(StdInSocketChannel):
     
     def call_handlers(self, msg):
         #io.rprint('[[Rep]]', msg) # dbg
@@ -114,8 +114,8 b' class BlockingHBSocketChannel(HBSocketChannel):'
 class BlockingKernelManager(KernelManager):
     
     # The classes to use for the various channels.
-    xreq_channel_class = Type(BlockingXReqSocketChannel)
+    shell_channel_class = Type(BlockingShellSocketChannel)
     sub_channel_class = Type(BlockingSubSocketChannel)
-    rep_channel_class = Type(BlockingRepSocketChannel)
+    stdin_channel_class = Type(BlockingStdInSocketChannel)
     hb_channel_class = Type(BlockingHBSocketChannel)
   

IPython/zmq/entry_point.py

@@ -9,159 +9,14 b' import socket'
 from subprocess import Popen, PIPE
 import sys
 
-# System library imports.
-import zmq
-
 # Local imports.
-from IPython.core.ultratb import FormattedTB
-from IPython.external.argparse import ArgumentParser
-from IPython.utils import io
-from IPython.utils.localinterfaces import LOCALHOST
-from displayhook import DisplayHook
-from heartbeat import Heartbeat
-from iostream import OutStream
-from parentpoller import ParentPollerUnix, ParentPollerWindows
-from session import Session
-
-
-def bind_port(socket, ip, port):
-    """ Binds the specified ZMQ socket. If the port is zero, a random port is
-    chosen. Returns the port that was bound.
-    """
-    connection = 'tcp://%s' % ip
-    if port <= 0:
-        port = socket.bind_to_random_port(connection)
-    else:
-        connection += ':%i' % port
-        socket.bind(connection)
-    return port
-
-
-def make_argument_parser():
-    """ Creates an ArgumentParser for the generic arguments supported by all 
-    kernel entry points.
-    """
-    parser = ArgumentParser()
-    parser.add_argument('--ip', type=str, default=LOCALHOST,
-                        help='set the kernel\'s IP address [default: local]')
-    parser.add_argument('--xrep', type=int, metavar='PORT', default=0,
-                        help='set the XREP channel port [default: random]')
-    parser.add_argument('--pub', type=int, metavar='PORT', default=0,
-                        help='set the PUB channel port [default: random]')
-    parser.add_argument('--req', type=int, metavar='PORT', default=0,
-                        help='set the REQ channel port [default: random]')
-    parser.add_argument('--hb', type=int, metavar='PORT', default=0,
-                        help='set the heartbeat port [default: random]')
-    parser.add_argument('--no-stdout', action='store_true',
-                        help='redirect stdout to the null device')
-    parser.add_argument('--no-stderr', action='store_true',
-                        help='redirect stderr to the null device')
-
-    if sys.platform == 'win32':
-        parser.add_argument('--interrupt', type=int, metavar='HANDLE', 
-                            default=0, help='interrupt this process when '
-                            'HANDLE is signaled')
-        parser.add_argument('--parent', type=int, metavar='HANDLE', 
-                            default=0, help='kill this process if the process '
-                            'with HANDLE dies')
-    else:
-        parser.add_argument('--parent', action='store_true', 
-                            help='kill this process if its parent dies')
-
-    return parser
-
-
-def make_kernel(namespace, kernel_factory, 
-                out_stream_factory=None, display_hook_factory=None):
-    """ Creates a kernel, redirects stdout/stderr, and installs a display hook
-    and exception handler.
-    """
-    # Re-direct stdout/stderr, if necessary.
-    if namespace.no_stdout or namespace.no_stderr:
-        blackhole = file(os.devnull, 'w')
-        if namespace.no_stdout:
-            sys.stdout = sys.__stdout__ = blackhole
-        if namespace.no_stderr:
-            sys.stderr = sys.__stderr__ = blackhole
-
-    # Install minimal exception handling
-    sys.excepthook = FormattedTB(mode='Verbose', color_scheme='NoColor', 
-                                 ostream=sys.__stdout__)
+from parentpoller import ParentPollerWindows
 
-    # Create a context, a session, and the kernel sockets.
-    io.raw_print("Starting the kernel at pid:", os.getpid())
-    context = zmq.Context()
-    # Uncomment this to try closing the context.
-    # atexit.register(context.close)
-    session = Session(username=u'kernel')
 
-    reply_socket = context.socket(zmq.XREP)
-    xrep_port = bind_port(reply_socket, namespace.ip, namespace.xrep)
-    io.raw_print("XREP Channel on port", xrep_port)
 
-    pub_socket = context.socket(zmq.PUB)
-    pub_port = bind_port(pub_socket, namespace.ip, namespace.pub)
-    io.raw_print("PUB Channel on port", pub_port)
-
-    req_socket = context.socket(zmq.XREQ)
-    req_port = bind_port(req_socket, namespace.ip, namespace.req)
-    io.raw_print("REQ Channel on port", req_port)
-
-    hb = Heartbeat(context, (namespace.ip, namespace.hb))
-    hb.start()
-    hb_port = hb.port
-    io.raw_print("Heartbeat REP Channel on port", hb_port)
-
-    # Helper to make it easier to connect to an existing kernel, until we have
-    # single-port connection negotiation fully implemented.
-    io.raw_print("To connect another client to this kernel, use:")
-    io.raw_print("-e --xreq {0} --sub {1} --rep {2} --hb {3}".format(
-        xrep_port, pub_port, req_port, hb_port))
-
-    # Redirect input streams and set a display hook.
-    if out_stream_factory:
-        sys.stdout = out_stream_factory(session, pub_socket, u'stdout')
-        sys.stderr = out_stream_factory(session, pub_socket, u'stderr')
-    if display_hook_factory:
-        sys.displayhook = display_hook_factory(session, pub_socket)
-
-    # Create the kernel.
-    kernel = kernel_factory(session=session, reply_socket=reply_socket, 
-                            pub_socket=pub_socket, req_socket=req_socket)
-    kernel.record_ports(xrep_port=xrep_port, pub_port=pub_port,
-                        req_port=req_port, hb_port=hb_port)
-    return kernel
-
-
-def start_kernel(namespace, kernel):
-    """ Starts a kernel.
-    """
-    # Configure this kernel process to poll the parent process, if necessary.
-    if sys.platform == 'win32':
-        if namespace.interrupt or namespace.parent:
-            poller = ParentPollerWindows(namespace.interrupt, namespace.parent)
-            poller.start()
-    elif namespace.parent:
-        poller = ParentPollerUnix()
-        poller.start()
-
-    # Start the kernel mainloop.
-    kernel.start()
-
-
-def make_default_main(kernel_factory):
-    """ Creates the simplest possible kernel entry point.
-    """
-    def main():
-        namespace = make_argument_parser().parse_args()
-        kernel = make_kernel(namespace, kernel_factory, OutStream, DisplayHook)
-        start_kernel(namespace, kernel)
-    return main
-
-
-def base_launch_kernel(code, xrep_port=0, pub_port=0, req_port=0, hb_port=0,
-                       stdin=None, stdout=None, stderr=None,
-                       executable=None, independent=False, extra_arguments=[]):
+def base_launch_kernel(code, shell_port=0, iopub_port=0, stdin_port=0, hb_port=0,
+                        ip=None, stdin=None, stdout=None, stderr=None,
+                        executable=None, independent=False, extra_arguments=[]):
     """ Launches a localhost kernel, binding to the specified ports.
 
     Parameters
@@ -169,18 +24,21 b' def base_launch_kernel(code, xrep_port=0, pub_port=0, req_port=0, hb_port=0,'
     code : str,
         A string of Python code that imports and executes a kernel entry point.
 
-    xrep_port : int, optional
+    shell_port : int, optional
         The port to use for XREP channel.
 
-    pub_port : int, optional
+    iopub_port : int, optional
         The port to use for the SUB channel.
 
-    req_port : int, optional
+    stdin_port : int, optional
         The port to use for the REQ (raw input) channel.
 
     hb_port : int, optional
         The port to use for the hearbeat REP channel.
 
+    ip  : str, optional
+        The ip address the kernel will bind to.
+
     stdin, stdout, stderr : optional (default None)
         Standards streams, as defined in subprocess.Popen.
 
@@ -199,13 +57,13 b' def base_launch_kernel(code, xrep_port=0, pub_port=0, req_port=0, hb_port=0,'
     Returns
     -------
     A tuple of form:
-        (kernel_process, xrep_port, pub_port, req_port)
+        (kernel_process, shell_port, iopub_port, stdin_port, hb_port)
     where kernel_process is a Popen object and the ports are integers.
     """
     # Find open ports as necessary.
     ports = []
-    ports_needed = int(xrep_port <= 0) + int(pub_port <= 0) + \
-                   int(req_port <= 0) + int(hb_port <= 0)
+    ports_needed = int(shell_port <= 0) + int(iopub_port <= 0) + \
+                   int(stdin_port <= 0) + int(hb_port <= 0)
     for i in xrange(ports_needed):
         sock = socket.socket()
         sock.bind(('', 0))
@@ -214,28 +72,31 b' def base_launch_kernel(code, xrep_port=0, pub_port=0, req_port=0, hb_port=0,'
         port = sock.getsockname()[1]
         sock.close()
         ports[i] = port
-    if xrep_port <= 0:
-        xrep_port = ports.pop(0)
-    if pub_port <= 0:
-        pub_port = ports.pop(0)
-    if req_port <= 0:
-        req_port = ports.pop(0)
+    if shell_port <= 0:
+        shell_port = ports.pop(0)
+    if iopub_port <= 0:
+        iopub_port = ports.pop(0)
+    if stdin_port <= 0:
+        stdin_port = ports.pop(0)
     if hb_port <= 0:
         hb_port = ports.pop(0)
 
     # Build the kernel launch command.
     if executable is None:
         executable = sys.executable
-    arguments = [ executable, '-c', code, '--xrep', str(xrep_port), 
-                  '--pub', str(pub_port), '--req', str(req_port),
-                  '--hb', str(hb_port) ]
+    arguments = [ executable, '-c', code, 'shell=%i'%shell_port,
+                  'iopub=%i'%iopub_port, 'stdin=%i'%stdin_port,
+                  'hb=%i'%hb_port
+    ]
+    if ip is not None:
+        arguments.append('ip=%s'%ip)
     arguments.extend(extra_arguments)
 
     # Spawn a kernel.
     if sys.platform == 'win32':
         # Create a Win32 event for interrupting the kernel.
         interrupt_event = ParentPollerWindows.create_interrupt_event()
-        arguments += [ '--interrupt', str(int(interrupt_event)) ]
+        arguments += [ 'interrupt=%i'%interrupt_event ]
 
         # If this process in running on pythonw, stdin, stdout, and stderr are
         # invalid. Popen will fail unless they are suitably redirected. We don't
@@ -273,7 +134,7 b' def base_launch_kernel(code, xrep_port=0, pub_port=0, req_port=0, hb_port=0,'
             handle = DuplicateHandle(pid, pid, pid, 0, 
                                      True, # Inheritable by new processes.
                                      DUPLICATE_SAME_ACCESS)
-            proc = Popen(arguments + ['--parent', str(int(handle))],
+            proc = Popen(arguments + ['parent=%i'%int(handle)],
                          stdin=_stdin, stdout=_stdout, stderr=_stderr)
 
         # Attach the interrupt event to the Popen objet so it can be used later.
@@ -293,7 +154,7 b' def base_launch_kernel(code, xrep_port=0, pub_port=0, req_port=0, hb_port=0,'
             proc = Popen(arguments, preexec_fn=lambda: os.setsid(),
                          stdin=stdin, stdout=stdout, stderr=stderr)
         else:
-            proc = Popen(arguments + ['--parent'],
+            proc = Popen(arguments + ['parent=1'],
                          stdin=stdin, stdout=stdout, stderr=stderr)
 
-    return proc, xrep_port, pub_port, req_port, hb_port
+    return proc, shell_port, iopub_port, stdin_port, hb_port

IPython/zmq/ipkernel.py

@@ -27,37 +27,25 b' import zmq'
 
 # Local imports.
 from IPython.config.configurable import Configurable
+from IPython.config.application import boolean_flag
+from IPython.core.application import ProfileDir
+from IPython.core.shellapp import (
+    InteractiveShellApp, shell_flags, shell_aliases
+)
 from IPython.utils import io
 from IPython.utils.jsonutil import json_clean
 from IPython.lib import pylabtools
-from IPython.utils.traitlets import Instance, Float
-from entry_point import (base_launch_kernel, make_argument_parser, make_kernel,
-                         start_kernel)
+from IPython.utils.traitlets import (
+    List, Instance, Float, Dict, Bool, Int, Unicode, CaselessStrEnum
+)
+
+from entry_point import base_launch_kernel
+from kernelapp import KernelApp, kernel_flags, kernel_aliases
 from iostream import OutStream
 from session import Session, Message
 from zmqshell import ZMQInteractiveShell
 
-#-----------------------------------------------------------------------------
-# Globals
-#-----------------------------------------------------------------------------
-
-# Module-level logger
-logger = logging.getLogger(__name__)
 
-# FIXME: this needs to be done more cleanly later, once we have proper
-# configuration support.  This is a library, so it shouldn't set a stream
-# handler, see:
-# http://docs.python.org/library/logging.html#configuring-logging-for-a-library
-# But this lets us at least do developer debugging for now by manually turning
-# it on/off.  And once we have full config support, the client entry points
-# will select their logging handlers, as well as passing to this library the
-# logging level.
-
-if 0:  # dbg - set to 1 to actually see the messages.
-    logger.addHandler(logging.StreamHandler())
-    logger.setLevel(logging.DEBUG)
-
-# /FIXME
 
 #-----------------------------------------------------------------------------
 # Main kernel class
@@ -71,9 +59,10 b' class Kernel(Configurable):'
 
     shell = Instance('IPython.core.interactiveshell.InteractiveShellABC')
     session = Instance(Session)
-    reply_socket = Instance('zmq.Socket')
-    pub_socket = Instance('zmq.Socket')
-    req_socket = Instance('zmq.Socket')
+    shell_socket = Instance('zmq.Socket')
+    iopub_socket = Instance('zmq.Socket')
+    stdin_socket = Instance('zmq.Socket')
+    log = Instance(logging.Logger)
 
     # Private interface
 
@@ -100,7 +89,8 b' class Kernel(Configurable):'
 
     # This is a dict of port number that the kernel is listening on. It is set
     # by record_ports and used by connect_request.
-    _recorded_ports = None
+    _recorded_ports = Dict()
+
 
 
     def __init__(self, **kwargs):
@@ -111,11 +101,11 b' class Kernel(Configurable):'
         atexit.register(self._at_shutdown)
 
         # Initialize the InteractiveShell subclass
-        self.shell = ZMQInteractiveShell.instance()
+        self.shell = ZMQInteractiveShell.instance(config=self.config)
         self.shell.displayhook.session = self.session
-        self.shell.displayhook.pub_socket = self.pub_socket
+        self.shell.displayhook.pub_socket = self.iopub_socket
         self.shell.display_pub.session = self.session
-        self.shell.display_pub.pub_socket = self.pub_socket
+        self.shell.display_pub.pub_socket = self.iopub_socket
 
         # TMP - hack while developing
         self.shell._reply_content = None
@@ -131,7 +121,7 b' class Kernel(Configurable):'
     def do_one_iteration(self):
         """Do one iteration of the kernel's evaluation loop.
         """
-        ident,msg = self.session.recv(self.reply_socket, zmq.NOBLOCK)
+        ident,msg = self.session.recv(self.shell_socket, zmq.NOBLOCK)
         if msg is None:
             return
         
@@ -143,21 +133,20 b' class Kernel(Configurable):'
         # Print some info about this message and leave a '--->' marker, so it's
         # easier to trace visually the message chain when debugging.  Each
         # handler prints its message at the end.
-        # Eventually we'll move these from stdout to a logger.
-        logger.debug('\n*** MESSAGE TYPE:'+str(msg['msg_type'])+'***')
-        logger.debug('   Content: '+str(msg['content'])+'\n   --->\n   ')
+        self.log.debug('\n*** MESSAGE TYPE:'+str(msg['msg_type'])+'***')
+        self.log.debug('   Content: '+str(msg['content'])+'\n   --->\n   ')
 
         # Find and call actual handler for message
         handler = self.handlers.get(msg['msg_type'], None)
         if handler is None:
-            logger.error("UNKNOWN MESSAGE TYPE:" +str(msg))
+            self.log.error("UNKNOWN MESSAGE TYPE:" +str(msg))
         else:
             handler(ident, msg)
             
         # Check whether we should exit, in case the incoming message set the
         # exit flag on
         if self.shell.exit_now:
-            logger.debug('\nExiting IPython kernel...')
+            self.log.debug('\nExiting IPython kernel...')
             # We do a normal, clean exit, which allows any actions registered
             # via atexit (such as history saving) to take place.
             sys.exit(0)
@@ -166,26 +155,27 b' class Kernel(Configurable):'
     def start(self):
         """ Start the kernel main loop.
         """
+        poller = zmq.Poller()
+        poller.register(self.shell_socket, zmq.POLLIN)
         while True:
             try:
-                time.sleep(self._poll_interval)
+                # scale by extra factor of 10, because there is no
+                # reason for this to be anything less than ~ 0.1s
+                # since it is a real poller and will respond
+                # to events immediately
+                poller.poll(10*1000*self._poll_interval)
                 self.do_one_iteration()
             except KeyboardInterrupt:
                 # Ctrl-C shouldn't crash the kernel
                 io.raw_print("KeyboardInterrupt caught in kernel")
 
-    def record_ports(self, xrep_port, pub_port, req_port, hb_port):
+    def record_ports(self, ports):
         """Record the ports that this kernel is using.
 
         The creator of the Kernel instance must call this methods if they
         want the :meth:`connect_request` method to return the port numbers.
         """
-        self._recorded_ports = {
-            'xrep_port' : xrep_port,
-            'pub_port' : pub_port,
-            'req_port' : req_port,
-            'hb_port' : hb_port
-        }
+        self._recorded_ports = ports
 
     #---------------------------------------------------------------------------
     # Kernel request handlers
@@ -194,11 +184,11 b' class Kernel(Configurable):'
     def _publish_pyin(self, code, parent):
         """Publish the code request on the pyin stream."""
 
-        pyin_msg = self.session.send(self.pub_socket, u'pyin',{u'code':code}, parent=parent)
+        pyin_msg = self.session.send(self.iopub_socket, u'pyin',{u'code':code}, parent=parent)
 
     def execute_request(self, ident, parent):
         
-        status_msg = self.session.send(self.pub_socket,
+        status_msg = self.session.send(self.iopub_socket,
             u'status',
             {u'execution_state':u'busy'},
             parent=parent
@@ -209,8 +199,8 b' class Kernel(Configurable):'
             code = content[u'code']
             silent = content[u'silent'] 
         except:
-            logger.error("Got bad msg: ")
-            logger.error(str(Message(parent)))
+            self.log.error("Got bad msg: ")
+            self.log.error(str(Message(parent)))
             return
 
         shell = self.shell # we'll need this a lot here
@@ -298,14 +288,14 b' class Kernel(Configurable):'
             time.sleep(self._execute_sleep)
         
         # Send the reply.
-        reply_msg = self.session.send(self.reply_socket, u'execute_reply',
+        reply_msg = self.session.send(self.shell_socket, u'execute_reply',
                                       reply_content, parent, ident=ident)
-        logger.debug(str(reply_msg))
+        self.log.debug(str(reply_msg))
 
         if reply_msg['content']['status'] == u'error':
             self._abort_queue()
 
-        status_msg = self.session.send(self.pub_socket,
+        status_msg = self.session.send(self.iopub_socket,
             u'status',
             {u'execution_state':u'idle'},
             parent=parent
@@ -316,17 +306,17 b' class Kernel(Configurable):'
         matches = {'matches' : matches,
                    'matched_text' : txt,
                    'status' : 'ok'}
-        completion_msg = self.session.send(self.reply_socket, 'complete_reply',
+        completion_msg = self.session.send(self.shell_socket, 'complete_reply',
                                            matches, parent, ident)
-        logger.debug(str(completion_msg))
+        self.log.debug(str(completion_msg))
 
     def object_info_request(self, ident, parent):
         object_info = self.shell.object_inspect(parent['content']['oname'])
         # Before we send this object over, we scrub it for JSON usage
         oinfo = json_clean(object_info)
-        msg = self.session.send(self.reply_socket, 'object_info_reply',
+        msg = self.session.send(self.shell_socket, 'object_info_reply',
                                 oinfo, parent, ident)
-        logger.debug(msg)
+        self.log.debug(msg)
 
     def history_request(self, ident, parent):
         # We need to pull these out, as passing **kwargs doesn't work with
@@ -353,18 +343,18 b' class Kernel(Configurable):'
         else:
             hist = []
         content = {'history' : list(hist)}
-        msg = self.session.send(self.reply_socket, 'history_reply',
+        msg = self.session.send(self.shell_socket, 'history_reply',
                                 content, parent, ident)
-        logger.debug(str(msg))
+        self.log.debug(str(msg))
 
     def connect_request(self, ident, parent):
         if self._recorded_ports is not None:
             content = self._recorded_ports.copy()
         else:
             content = {}
-        msg = self.session.send(self.reply_socket, 'connect_reply',
+        msg = self.session.send(self.shell_socket, 'connect_reply',
                                 content, parent, ident)
-        logger.debug(msg)
+        self.log.debug(msg)
 
     def shutdown_request(self, ident, parent):
         self.shell.exit_now = True
@@ -377,19 +367,19 b' class Kernel(Configurable):'
 
     def _abort_queue(self):
         while True:
-            ident,msg = self.session.recv(self.reply_socket, zmq.NOBLOCK)
+            ident,msg = self.session.recv(self.shell_socket, zmq.NOBLOCK)
             if msg is None:
                 break
             else:
                 assert ident is not None, \
                        "Unexpected missing message part."
 
-            logger.debug("Aborting:\n"+str(Message(msg)))
+            self.log.debug("Aborting:\n"+str(Message(msg)))
             msg_type = msg['msg_type']
             reply_type = msg_type.split('_')[0] + '_reply'
-            reply_msg = self.session.send(self.reply_socket, reply_type, 
+            reply_msg = self.session.send(self.shell_socket, reply_type,
                     {'status' : 'aborted'}, msg, ident=ident)
-            logger.debug(reply_msg)
+            self.log.debug(reply_msg)
             # We need to wait a bit for requests to come in. This can probably
             # be set shorter for true asynchronous clients.
             time.sleep(0.1)
@@ -401,15 +391,15 b' class Kernel(Configurable):'
 
         # Send the input request.
         content = dict(prompt=prompt)
-        msg = self.session.send(self.req_socket, u'input_request', content, parent)
+        msg = self.session.send(self.stdin_socket, u'input_request', content, parent)
 
         # Await a response.
-        ident, reply = self.session.recv(self.req_socket, 0)
+        ident, reply = self.session.recv(self.stdin_socket, 0)
         try:
             value = reply['content']['value']
         except:
-            logger.error("Got bad raw_input reply: ")
-            logger.error(str(Message(parent)))
+            self.log.error("Got bad raw_input reply: ")
+            self.log.error(str(Message(parent)))
             value = ''
         return value
     
@@ -461,9 +451,9 b' class Kernel(Configurable):'
         """
         # io.rprint("Kernel at_shutdown") # dbg
         if self._shutdown_message is not None:
-            self.session.send(self.reply_socket, self._shutdown_message)
-            self.session.send(self.pub_socket, self._shutdown_message)
-            logger.debug(str(self._shutdown_message))
+            self.session.send(self.shell_socket, self._shutdown_message)
+            self.session.send(self.iopub_socket, self._shutdown_message)
+            self.log.debug(str(self._shutdown_message))
             # A very short sleep to give zmq time to flush its message buffers
             # before Python truly shuts down.
             time.sleep(0.01)
@@ -569,120 +559,114 b' class GTKKernel(Kernel):'
 
 
 #-----------------------------------------------------------------------------
-# Kernel main and launch functions
+# Aliases and Flags for the IPKernelApp
 #-----------------------------------------------------------------------------
 
-def launch_kernel(ip=None, xrep_port=0, pub_port=0, req_port=0, hb_port=0,
-                  stdin=None, stdout=None, stderr=None,
-                  executable=None, independent=False, pylab=False, colors=None):
-    """Launches a localhost kernel, binding to the specified ports.
+flags = dict(kernel_flags)
+flags.update(shell_flags)
 
-    Parameters
-    ----------
-    ip  : str, optional
-        The ip address the kernel will bind to.
-    
-    xrep_port : int, optional
-        The port to use for XREP channel.
+addflag = lambda *args: flags.update(boolean_flag(*args))
 
-    pub_port : int, optional
-        The port to use for the SUB channel.
+flags['pylab'] = (
+    {'IPKernelApp' : {'pylab' : 'auto'}},
+    """Pre-load matplotlib and numpy for interactive use with
+    the default matplotlib backend."""
+)
 
-    req_port : int, optional
-        The port to use for the REQ (raw input) channel.
+aliases = dict(kernel_aliases)
+aliases.update(shell_aliases)
 
-    hb_port : int, optional
-        The port to use for the hearbeat REP channel.
+# it's possible we don't want short aliases for *all* of these:
+aliases.update(dict(
+    pylab='IPKernelApp.pylab',
+))
 
-    stdin, stdout, stderr : optional (default None)
-        Standards streams, as defined in subprocess.Popen.
+#-----------------------------------------------------------------------------
+# The IPKernelApp class
+#-----------------------------------------------------------------------------
 
-    executable : str, optional (default sys.executable)
-        The Python executable to use for the kernel process.
+class IPKernelApp(KernelApp, InteractiveShellApp):
+    name = 'ipkernel'
+
+    aliases = Dict(aliases)
+    flags = Dict(flags)
+    classes = [Kernel, ZMQInteractiveShell, ProfileDir, Session]
+    # configurables
+    pylab = CaselessStrEnum(['tk', 'qt', 'wx', 'gtk', 'osx', 'inline', 'auto'],
+        config=True,
+        help="""Pre-load matplotlib and numpy for interactive use,
+        selecting a particular matplotlib backend and loop integration.
+        """
+    )
+    def initialize(self, argv=None):
+        super(IPKernelApp, self).initialize(argv)
+        self.init_shell()
+        self.init_extensions()
+        self.init_code()
+
+    def init_kernel(self):
+        kernel_factory = Kernel
+
+        kernel_map = {
+            'qt' : QtKernel,
+            'qt4': QtKernel,
+            'inline': Kernel,
+            'osx': TkKernel,
+            'wx' : WxKernel,
+            'tk' : TkKernel,
+            'gtk': GTKKernel,
+        }
+
+        if self.pylab:
+            key = None if self.pylab == 'auto' else self.pylab
+            gui, backend = pylabtools.find_gui_and_backend(key)
+            kernel_factory = kernel_map.get(gui)
+            if kernel_factory is None:
+                raise ValueError('GUI is not supported: %r' % gui)
+            pylabtools.activate_matplotlib(backend)
+
+        kernel = kernel_factory(config=self.config, session=self.session,
+                                shell_socket=self.shell_socket,
+                                iopub_socket=self.iopub_socket,
+                                stdin_socket=self.stdin_socket,
+                                log=self.log
+        )
+        self.kernel = kernel
+        kernel.record_ports(self.ports)
+
+        if self.pylab:
+            pylabtools.import_pylab(kernel.shell.user_ns, backend,
+                                    shell=kernel.shell)
+    
+    def init_shell(self):
+        self.shell = self.kernel.shell
 
-    independent : bool, optional (default False) 
-        If set, the kernel process is guaranteed to survive if this process
-        dies. If not set, an effort is made to ensure that the kernel is killed
-        when this process dies. Note that in this case it is still good practice
-        to kill kernels manually before exiting.
 
-    pylab : bool or string, optional (default False)
-        If not False, the kernel will be launched with pylab enabled. If a
-        string is passed, matplotlib will use the specified backend. Otherwise,
-        matplotlib's default backend will be used.
+#-----------------------------------------------------------------------------
+# Kernel main and launch functions
+#-----------------------------------------------------------------------------
 
-    colors : None or string, optional (default None)
-        If not None, specify the color scheme. One of (NoColor, LightBG, Linux)
+def launch_kernel(*args, **kwargs):
+    """Launches a localhost IPython kernel, binding to the specified ports.
+
+    This function simply calls entry_point.base_launch_kernel with the right first
+    command to start an ipkernel.  See base_launch_kernel for arguments.
 
     Returns
     -------
     A tuple of form:
-        (kernel_process, xrep_port, pub_port, req_port)
+        (kernel_process, shell_port, iopub_port, stdin_port, hb_port)
     where kernel_process is a Popen object and the ports are integers.
     """
-    extra_arguments = []
-    if pylab:
-        extra_arguments.append('--pylab')
-        if isinstance(pylab, basestring):
-            extra_arguments.append(pylab)
-    if ip is not None:
-        extra_arguments.append('--ip')
-        if isinstance(ip, basestring):
-            extra_arguments.append(ip)
-    if colors is not None:
-        extra_arguments.append('--colors')
-        extra_arguments.append(colors)
     return base_launch_kernel('from IPython.zmq.ipkernel import main; main()',
-                              xrep_port, pub_port, req_port, hb_port, 
-                              stdin, stdout, stderr,
-                              executable, independent, extra_arguments)
+                              *args, **kwargs)
 
 
 def main():
-    """ The IPython kernel main entry point.
-    """
-    parser = make_argument_parser()
-    parser.add_argument('--pylab', type=str, metavar='GUI', nargs='?', 
-                        const='auto', help = \
-"Pre-load matplotlib and numpy for interactive use. If GUI is not \
-given, the GUI backend is matplotlib's, otherwise use one of: \
-['tk', 'gtk', 'qt', 'wx', 'osx', 'inline'].")
-    parser.add_argument('--colors',
-        type=str, dest='colors',
-        help="Set the color scheme (NoColor, Linux, and LightBG).",
-        metavar='ZMQInteractiveShell.colors')
-    namespace = parser.parse_args()
-
-    kernel_class = Kernel
-
-    kernel_classes = {
-        'qt' : QtKernel,
-        'qt4': QtKernel,
-        'inline': Kernel,
-        'osx': TkKernel,
-        'wx' : WxKernel,
-        'tk' : TkKernel,
-        'gtk': GTKKernel,
-    }
-    if namespace.pylab:
-        if namespace.pylab == 'auto':
-            gui, backend = pylabtools.find_gui_and_backend()
-        else:
-            gui, backend = pylabtools.find_gui_and_backend(namespace.pylab)
-        kernel_class = kernel_classes.get(gui)
-        if kernel_class is None:
-            raise ValueError('GUI is not supported: %r' % gui)
-        pylabtools.activate_matplotlib(backend)
-    if namespace.colors:
-        ZMQInteractiveShell.colors=namespace.colors
-
-    kernel = make_kernel(namespace, kernel_class, OutStream)
-
-    if namespace.pylab:
-        pylabtools.import_pylab(kernel.shell.user_ns, backend,
-                                shell=kernel.shell)
-
-    start_kernel(namespace, kernel)
+    """Run an IPKernel as an application"""
+    app = IPKernelApp.instance()
+    app.initialize()
+    app.start()
 
 
 if __name__ == '__main__':

IPython/zmq/kernelmanager.py

@@ -32,6 +32,7 b' from zmq import POLLIN, POLLOUT, POLLERR'
 from zmq.eventloop import ioloop
 
 # Local imports.
+from IPython.config.loader import Config
 from IPython.utils import io
 from IPython.utils.localinterfaces import LOCALHOST, LOCAL_IPS
 from IPython.utils.traitlets import HasTraits, Any, Instance, Type, TCPAddress
@@ -77,7 +78,7 b' def validate_string_dict(dct):'
 # ZMQ Socket Channel classes
 #-----------------------------------------------------------------------------
 
-class ZmqSocketChannel(Thread):
+class ZMQSocketChannel(Thread):
     """The base class for the channels that use ZMQ sockets.
     """
     context = None
@@ -99,7 +100,7 b' class ZmqSocketChannel(Thread):'
         address : tuple
             Standard (ip, port) tuple that the kernel is listening on.
         """
-        super(ZmqSocketChannel, self).__init__()
+        super(ZMQSocketChannel, self).__init__()
         self.daemon = True
 
         self.context = context
@@ -173,14 +174,14 b' class ZmqSocketChannel(Thread):'
         self.ioloop.add_callback(drop_io_state_callback)
 
 
-class XReqSocketChannel(ZmqSocketChannel):
+class ShellSocketChannel(ZMQSocketChannel):
     """The XREQ channel for issues request/replies to the kernel.
     """
 
     command_queue = None
 
     def __init__(self, context, session, address):
-        super(XReqSocketChannel, self).__init__(context, session, address)
+        super(ShellSocketChannel, self).__init__(context, session, address)
         self.command_queue = Queue()
         self.ioloop = ioloop.IOLoop()
 
@@ -196,7 +197,7 b' class XReqSocketChannel(ZmqSocketChannel):'
 
     def stop(self):
         self.ioloop.stop()
-        super(XReqSocketChannel, self).stop()
+        super(ShellSocketChannel, self).stop()
 
     def call_handlers(self, msg):
         """This method is called in the ioloop thread when a message arrives.
@@ -382,7 +383,7 b' class XReqSocketChannel(ZmqSocketChannel):'
         self.add_io_state(POLLOUT)
 
 
-class SubSocketChannel(ZmqSocketChannel):
+class SubSocketChannel(ZMQSocketChannel):
     """The SUB channel which listens for messages that the kernel publishes.
     """
 
@@ -469,13 +470,13 b' class SubSocketChannel(ZmqSocketChannel):'
         self._flushed = True
 
 
-class RepSocketChannel(ZmqSocketChannel):
+class StdInSocketChannel(ZMQSocketChannel):
     """A reply channel to handle raw_input requests that the kernel makes."""
 
     msg_queue = None
 
     def __init__(self, context, session, address):
-        super(RepSocketChannel, self).__init__(context, session, address)
+        super(StdInSocketChannel, self).__init__(context, session, address)
         self.ioloop = ioloop.IOLoop()
         self.msg_queue = Queue()
 
@@ -491,7 +492,7 b' class RepSocketChannel(ZmqSocketChannel):'
 
     def stop(self):
         self.ioloop.stop()
-        super(RepSocketChannel, self).stop()
+        super(StdInSocketChannel, self).stop()
 
     def call_handlers(self, msg):
         """This method is called in the ioloop thread when a message arrives.
@@ -540,7 +541,7 b' class RepSocketChannel(ZmqSocketChannel):'
         self.add_io_state(POLLOUT)
 
 
-class HBSocketChannel(ZmqSocketChannel):
+class HBSocketChannel(ZMQSocketChannel):
     """The heartbeat channel which monitors the kernel heartbeat.
 
     Note that the heartbeat channel is paused by default. As long as you start
@@ -676,44 +677,51 b' class KernelManager(HasTraits):'
     The REP channel is for the kernel to request stdin (raw_input) from the
     frontend.
     """
+    # config object for passing to child configurables
+    config = Instance(Config)
+    
     # The PyZMQ Context to use for communication with the kernel.
-    context = Instance(zmq.Context,(),{})
+    context = Instance(zmq.Context)
+    def _context_default(self):
+        return zmq.Context.instance()
 
     # The Session to use for communication with the kernel.
-    session = Instance(Session,(),{})
+    session = Instance(Session)
 
     # The kernel process with which the KernelManager is communicating.
     kernel = Instance(Popen)
 
     # The addresses for the communication channels. 
-    xreq_address = TCPAddress((LOCALHOST, 0))
+    shell_address = TCPAddress((LOCALHOST, 0))
     sub_address = TCPAddress((LOCALHOST, 0))
-    rep_address = TCPAddress((LOCALHOST, 0))
+    stdin_address = TCPAddress((LOCALHOST, 0))
     hb_address = TCPAddress((LOCALHOST, 0))
 
     # The classes to use for the various channels.
-    xreq_channel_class = Type(XReqSocketChannel)
+    shell_channel_class = Type(ShellSocketChannel)
     sub_channel_class = Type(SubSocketChannel)
-    rep_channel_class = Type(RepSocketChannel)
+    stdin_channel_class = Type(StdInSocketChannel)
     hb_channel_class = Type(HBSocketChannel)
 
     # Protected traits.
     _launch_args = Any
-    _xreq_channel = Any
+    _shell_channel = Any
     _sub_channel = Any
-    _rep_channel = Any
+    _stdin_channel = Any
     _hb_channel = Any
 
     def __init__(self, **kwargs):
         super(KernelManager, self).__init__(**kwargs)
+        if self.session is None:
+            self.session = Session(config=self.config)
         # Uncomment this to try closing the context.
-        # atexit.register(self.context.close)
+        # atexit.register(self.context.term)
 
     #--------------------------------------------------------------------------
     # Channel management methods:
     #--------------------------------------------------------------------------
 
-    def start_channels(self, xreq=True, sub=True, rep=True, hb=True):
+    def start_channels(self, shell=True, sub=True, stdin=True, hb=True):
         """Starts the channels for this kernel.
 
         This will create the channels if they do not exist and then start
@@ -721,32 +729,32 b' class KernelManager(HasTraits):'
         must first call :method:`start_kernel`. If the channels have been
         stopped and you call this, :class:`RuntimeError` will be raised.
         """
-        if xreq:
-            self.xreq_channel.start()
+        if shell:
+            self.shell_channel.start()
         if sub:
             self.sub_channel.start()
-        if rep:
-            self.rep_channel.start()
+        if stdin:
+            self.stdin_channel.start()
         if hb:
             self.hb_channel.start()
 
     def stop_channels(self):
         """Stops all the running channels for this kernel.
         """
-        if self.xreq_channel.is_alive():
-            self.xreq_channel.stop()
+        if self.shell_channel.is_alive():
+            self.shell_channel.stop()
         if self.sub_channel.is_alive():
             self.sub_channel.stop()
-        if self.rep_channel.is_alive():
-            self.rep_channel.stop()
+        if self.stdin_channel.is_alive():
+            self.stdin_channel.stop()
         if self.hb_channel.is_alive():
             self.hb_channel.stop()
 
     @property
     def channels_running(self):
         """Are any of the channels created and running?"""
-        return (self.xreq_channel.is_alive() or self.sub_channel.is_alive() or
-                self.rep_channel.is_alive() or self.hb_channel.is_alive())
+        return (self.shell_channel.is_alive() or self.sub_channel.is_alive() or
+                self.stdin_channel.is_alive() or self.hb_channel.is_alive())
 
     #--------------------------------------------------------------------------
     # Kernel process management methods:
@@ -766,10 +774,10 b' class KernelManager(HasTraits):'
         **kw : optional
              See respective options for IPython and Python kernels.
         """
-        xreq, sub, rep, hb = self.xreq_address, self.sub_address, \
-            self.rep_address, self.hb_address
-        if xreq[0] not in LOCAL_IPS or sub[0] not in LOCAL_IPS or \
-                rep[0] not in LOCAL_IPS or hb[0] not in LOCAL_IPS:
+        shell, sub, stdin, hb = self.shell_address, self.sub_address, \
+            self.stdin_address, self.hb_address
+        if shell[0] not in LOCAL_IPS or sub[0] not in LOCAL_IPS or \
+                stdin[0] not in LOCAL_IPS or hb[0] not in LOCAL_IPS:
             raise RuntimeError("Can only launch a kernel on a local interface. "
                                "Make sure that the '*_address' attributes are "
                                "configured properly. "
@@ -782,11 +790,11 b' class KernelManager(HasTraits):'
         else:
             from pykernel import launch_kernel
         self.kernel, xrep, pub, req, _hb = launch_kernel(
-            xrep_port=xreq[1], pub_port=sub[1], 
-            req_port=rep[1], hb_port=hb[1], **kw)
-        self.xreq_address = (xreq[0], xrep)
+            shell_port=shell[1], iopub_port=sub[1],
+            stdin_port=stdin[1], hb_port=hb[1], **kw)
+        self.shell_address = (shell[0], xrep)
         self.sub_address = (sub[0], pub)
-        self.rep_address = (rep[0], req)
+        self.stdin_address = (stdin[0], req)
         self.hb_address = (hb[0], _hb)
 
     def shutdown_kernel(self, restart=False):
@@ -805,7 +813,7 b' class KernelManager(HasTraits):'
         # Don't send any additional kernel kill messages immediately, to give
         # the kernel a chance to properly execute shutdown actions. Wait for at
         # most 1s, checking every 0.1s.
-        self.xreq_channel.shutdown(restart=restart)
+        self.shell_channel.shutdown(restart=restart)
         for i in range(10):
             if self.is_alive:
                 time.sleep(0.1)
@@ -931,13 +939,13 b' class KernelManager(HasTraits):'
     #--------------------------------------------------------------------------
 
     @property
-    def xreq_channel(self):
+    def shell_channel(self):
         """Get the REQ socket channel object to make requests of the kernel."""
-        if self._xreq_channel is None:
-            self._xreq_channel = self.xreq_channel_class(self.context, 
+        if self._shell_channel is None:
+            self._shell_channel = self.shell_channel_class(self.context,
                                                          self.session,
-                                                         self.xreq_address)
-        return self._xreq_channel
+                                                         self.shell_address)
+        return self._shell_channel
 
     @property
     def sub_channel(self):
@@ -949,13 +957,13 b' class KernelManager(HasTraits):'
         return self._sub_channel
 
     @property
-    def rep_channel(self):
+    def stdin_channel(self):
         """Get the REP socket channel object to handle stdin (raw_input)."""
-        if self._rep_channel is None:
-            self._rep_channel = self.rep_channel_class(self.context, 
+        if self._stdin_channel is None:
+            self._stdin_channel = self.stdin_channel_class(self.context,
                                                        self.session,
-                                                       self.rep_address)
-        return self._rep_channel
+                                                       self.stdin_address)
+        return self._stdin_channel
 
     @property
     def hb_channel(self):

IPython/zmq/pykernel.py

@@ -25,10 +25,11 b' import traceback'
 import zmq
 
 # Local imports.
-from IPython.utils.traitlets import HasTraits, Instance, Float
+from IPython.utils.traitlets import HasTraits, Instance, Dict, Float
 from completer import KernelCompleter
-from entry_point import base_launch_kernel, make_default_main
+from entry_point import base_launch_kernel
 from session import Session, Message
+from kernelapp import KernelApp
 
 #-----------------------------------------------------------------------------
 # Main kernel class
@@ -49,16 +50,17 b' class Kernel(HasTraits):'
 
     # This is a dict of port number that the kernel is listening on. It is set
     # by record_ports and used by connect_request.
-    _recorded_ports = None
+    _recorded_ports = Dict()
 
     #---------------------------------------------------------------------------
     # Kernel interface
     #---------------------------------------------------------------------------
 
     session = Instance(Session)
-    reply_socket = Instance('zmq.Socket')
-    pub_socket = Instance('zmq.Socket')
-    req_socket = Instance('zmq.Socket')
+    shell_socket = Instance('zmq.Socket')
+    iopub_socket = Instance('zmq.Socket')
+    stdin_socket = Instance('zmq.Socket')
+    log = Instance('logging.Logger')
 
     def __init__(self, **kwargs):
         super(Kernel, self).__init__(**kwargs)
@@ -78,29 +80,23 b' class Kernel(HasTraits):'
         """ Start the kernel main loop.
         """
         while True:
-            ident,msg = self.session.recv(self.reply_socket,0)
+            ident,msg = self.session.recv(self.shell_socket,0)
             assert ident is not None, "Missing message part."
             omsg = Message(msg)
-            print>>sys.__stdout__
-            print>>sys.__stdout__, omsg
+            self.log.debug(str(omsg))
             handler = self.handlers.get(omsg.msg_type, None)
             if handler is None:
-                print >> sys.__stderr__, "UNKNOWN MESSAGE TYPE:", omsg
+                self.log.error("UNKNOWN MESSAGE TYPE: %s"%omsg)
             else:
                 handler(ident, omsg)
 
-    def record_ports(self, xrep_port, pub_port, req_port, hb_port):
+    def record_ports(self, ports):
         """Record the ports that this kernel is using.
 
         The creator of the Kernel instance must call this methods if they
         want the :meth:`connect_request` method to return the port numbers.
         """
-        self._recorded_ports = {
-            'xrep_port' : xrep_port,
-            'pub_port' : pub_port,
-            'req_port' : req_port,
-            'hb_port' : hb_port
-        }
+        self._recorded_ports = ports
 
     #---------------------------------------------------------------------------
     # Kernel request handlers
@@ -110,10 +106,9 b' class Kernel(HasTraits):'
         try:
             code = parent[u'content'][u'code']
         except:
-            print>>sys.__stderr__, "Got bad msg: "
-            print>>sys.__stderr__, Message(parent)
+            self.log.error("Got bad msg: %s"%Message(parent))
             return
-        pyin_msg = self.session.send(self.pub_socket, u'pyin',{u'code':code}, parent=parent)
+        pyin_msg = self.session.send(self.iopub_socket, u'pyin',{u'code':code}, parent=parent)
 
         try:
             comp_code = self.compiler(code, '<zmq-kernel>')
@@ -138,7 +133,7 b' class Kernel(HasTraits):'
                 u'ename' : unicode(etype.__name__),
                 u'evalue' : unicode(evalue)
             }
-            exc_msg = self.session.send(self.pub_socket, u'pyerr', exc_content, parent)
+            exc_msg = self.session.send(self.iopub_socket, u'pyerr', exc_content, parent)
             reply_content = exc_content
         else:
             reply_content = { 'status' : 'ok', 'payload' : {} }
@@ -153,32 +148,32 b' class Kernel(HasTraits):'
             time.sleep(self._execute_sleep)
 
         # Send the reply.
-        reply_msg = self.session.send(self.reply_socket, u'execute_reply', reply_content, parent, ident=ident)
-        print>>sys.__stdout__, Message(reply_msg)
+        reply_msg = self.session.send(self.shell_socket, u'execute_reply', reply_content, parent, ident=ident)
+        self.log.debug(Message(reply_msg))
         if reply_msg['content']['status'] == u'error':
             self._abort_queue()
 
     def complete_request(self, ident, parent):
         matches = {'matches' : self._complete(parent),
                    'status' : 'ok'}
-        completion_msg = self.session.send(self.reply_socket, 'complete_reply',
+        completion_msg = self.session.send(self.shell_socket, 'complete_reply',
                                            matches, parent, ident)
-        print >> sys.__stdout__, completion_msg
+        self.log.debug(completion_msg)
 
     def object_info_request(self, ident, parent):
         context = parent['content']['oname'].split('.')
         object_info = self._object_info(context)
-        msg = self.session.send(self.reply_socket, 'object_info_reply',
+        msg = self.session.send(self.shell_socket, 'object_info_reply',
                                 object_info, parent, ident)
-        print >> sys.__stdout__, msg
+        self.log.debug(msg)
 
     def shutdown_request(self, ident, parent):
         content = dict(parent['content'])
-        msg = self.session.send(self.reply_socket, 'shutdown_reply',
+        msg = self.session.send(self.shell_socket, 'shutdown_reply',
                                 content, parent, ident)
-        msg = self.session.send(self.pub_socket, 'shutdown_reply',
+        msg = self.session.send(self.iopub_socket, 'shutdown_reply',
                                 content, parent, ident)
-        print >> sys.__stdout__, msg
+        self.log.debug(msg)
         time.sleep(0.1)
         sys.exit(0)
 
@@ -188,17 +183,17 b' class Kernel(HasTraits):'
 
     def _abort_queue(self):
         while True:
-            ident,msg = self.session.recv(self.reply_socket, zmq.NOBLOCK)
+            ident,msg = self.session.recv(self.shell_socket, zmq.NOBLOCK)
             if msg is None:
+                # msg=None on EAGAIN
                 break
             else:
-                assert ident is not None, "Unexpected missing message part."
-            print>>sys.__stdout__, "Aborting:"
-            print>>sys.__stdout__, Message(msg)
+                assert ident is not None, "Missing message part."
+            self.log.debug("Aborting: %s"%Message(msg))
             msg_type = msg['msg_type']
             reply_type = msg_type.split('_')[0] + '_reply'
-            reply_msg = self.session.send(self.reply_socket, reply_type, {'status':'aborted'}, msg, ident=ident)
-            print>>sys.__stdout__, Message(reply_msg)
+            reply_msg = self.session.send(self.shell_socket, reply_type, {'status':'aborted'}, msg, ident=ident)
+            self.log.debug(Message(reply_msg))
             # We need to wait a bit for requests to come in. This can probably
             # be set shorter for true asynchronous clients.
             time.sleep(0.1)
@@ -210,15 +205,14 b' class Kernel(HasTraits):'
 
         # Send the input request.
         content = dict(prompt=prompt)
-        msg = self.session.send(self.req_socket, u'input_request', content, parent)
+        msg = self.session.send(self.stdin_socket, u'input_request', content, parent)
 
         # Await a response.
-        ident,reply = self.session.recv(self.req_socket, 0)
+        ident,reply = self.session.recv(self.stdin_socket, 0)
         try:
             value = reply['content']['value']
         except:
-            print>>sys.__stderr__, "Got bad raw_input reply: "
-            print>>sys.__stderr__, Message(parent)
+            self.log.error("Got bad raw_input reply: %s"%Message(parent))
             value = ''
         return value
 
@@ -259,58 +253,26 b' class Kernel(HasTraits):'
 # Kernel main and launch functions
 #-----------------------------------------------------------------------------
 
-def launch_kernel(ip=None, xrep_port=0, pub_port=0, req_port=0, hb_port=0,
-                  stdin=None, stdout=None, stderr=None,
-                  executable=None, independent=False):
-    """ Launches a localhost kernel, binding to the specified ports.
-
-    Parameters
-    ----------
-    ip  : str, optional
-        The ip address the kernel will bind to.
+def launch_kernel(*args, **kwargs):
+    """ Launches a simple Python kernel, binding to the specified ports.
     
-    xrep_port : int, optional
-        The port to use for XREP channel.
-
-    pub_port : int, optional
-        The port to use for the SUB channel.
-
-    req_port : int, optional
-        The port to use for the REQ (raw input) channel.
-
-    hb_port : int, optional
-        The port to use for the hearbeat REP channel.
-
-    stdin, stdout, stderr : optional (default None)
-        Standards streams, as defined in subprocess.Popen.
-
-    executable : str, optional (default sys.executable)
-        The Python executable to use for the kernel process.
-
-    independent : bool, optional (default False) 
-        If set, the kernel process is guaranteed to survive if this process
-        dies. If not set, an effort is made to ensure that the kernel is killed
-        when this process dies. Note that in this case it is still good practice
-        to kill kernels manually before exiting.
+    This function simply calls entry_point.base_launch_kernel with the right first
+    command to start a pykernel.  See base_launch_kernel for arguments.
 
     Returns
     -------
     A tuple of form:
-        (kernel_process, xrep_port, pub_port, req_port)
+        (kernel_process, xrep_port, pub_port, req_port, hb_port)
     where kernel_process is a Popen object and the ports are integers.
     """
-    extra_arguments = []
-    if ip is not None:
-        extra_arguments.append('--ip')
-        if isinstance(ip, basestring):
-            extra_arguments.append(ip)
-    
     return base_launch_kernel('from IPython.zmq.pykernel import main; main()',
-                              xrep_port, pub_port, req_port, hb_port,
-                              stdin, stdout, stderr,
-                              executable, independent, extra_arguments)
+                                *args, **kwargs)
 
-main = make_default_main(Kernel)
+def main():
+    """Run a PyKernel as an application"""
+    app = KernelApp.instance()
+    app.initialize()
+    app.start()
 
 if __name__ == '__main__':
     main()

IPython/zmq/pylab/backend_inline.py

@@ -10,12 +10,46 b' import sys'
 
 # Third-party imports
 import matplotlib
-from matplotlib.backends.backend_svg import new_figure_manager
+from matplotlib.backends.backend_agg import new_figure_manager
 from matplotlib._pylab_helpers import Gcf
 
 # Local imports.
+from IPython.config.configurable import SingletonConfigurable
 from IPython.core.displaypub import publish_display_data
-from IPython.lib.pylabtools import figure_to_svg
+from IPython.lib.pylabtools import print_figure, select_figure_format
+from IPython.utils.traitlets import Dict, Instance, CaselessStrEnum
+#-----------------------------------------------------------------------------
+# Configurable for inline backend options
+#-----------------------------------------------------------------------------
+
+class InlineBackendConfig(SingletonConfigurable):
+    """An object to store configuration of the inline backend."""
+
+    # The typical default figure size is too large for inline use,
+    # so we shrink the figure size to 6x4, and tweak fonts to
+    # make that fit.  This is configurable via Global.pylab_inline_rc,
+    # or rather it will be once the zmq kernel is hooked up to
+    # the config system.
+    rc = Dict({'figure.figsize': (6.0,4.0),
+        # 12pt labels get cutoff on 6x4 logplots, so use 10pt.
+        'font.size': 10,
+        # 10pt still needs a little more room on the xlabel:
+        'figure.subplot.bottom' : .125
+        }, config=True,
+        help="""Subset of matplotlib rcParams that should be different for the
+        inline backend."""
+    )
+    figure_format = CaselessStrEnum(['svg', 'png'], default_value='png', config=True,
+        help="The image format for figures with the inline backend.")
+
+    def _figure_format_changed(self, name, old, new):
+        if self.shell is None:
+            return
+        else:
+            select_figure_format(self.shell, new)
+
+    shell = Instance('IPython.core.interactiveshell.InteractiveShellABC')
+
 
 #-----------------------------------------------------------------------------
 # Functions
@@ -32,7 +66,7 b' def show(close=True):'
       removed from the internal list of figures.
     """
     for figure_manager in Gcf.get_all_fig_managers():
-        send_svg_figure(figure_manager.canvas.figure)
+        send_figure(figure_manager.canvas.figure)
     if close:
         matplotlib.pyplot.close('all')
 
@@ -50,8 +84,8 b' def draw_if_interactive():'
     show._draw_called = True
 
 
-def flush_svg():
-    """Call show, close all open figures, sending all SVG images.
+def flush_figures():
+    """Call show, close all open figures, sending all figure images.
 
     This is meant to be called automatically and will call show() if, during
     prior code execution, there had been any calls to draw_if_interactive.
@@ -61,21 +95,23 b' def flush_svg():'
         show._draw_called = False
 
 
-def send_svg_figure(fig):
-    """Draw the current figure and send it as an SVG payload.
+def send_figure(fig):
+    """Draw the current figure and send it as a PNG payload.
     """
     # For an empty figure, don't even bother calling figure_to_svg, to avoid
     # big blank spaces in the qt console
     if not fig.axes:
         return
-
-    svg = figure_to_svg(fig)
+    fmt = InlineBackendConfig.instance().figure_format
+    data = print_figure(fig, fmt)
+    mimetypes = { 'png' : 'image/png', 'svg' : 'image/svg+xml' }
+    mime = mimetypes[fmt]
     # flush text streams before sending figures, helps a little with output
     # synchronization in the console (though it's a bandaid, not a real sln)
     sys.stdout.flush(); sys.stderr.flush()
     publish_display_data(
-        'IPython.zmq.pylab.backend_inline.send_svg_figure',
+        'IPython.zmq.pylab.backend_inline.send_figure',
         'Matplotlib Plot',
-        {'image/svg+xml' : svg}
+        {mime : data}
     )
 

IPython/zmq/session.py

@@ -1,10 +1,119 b''
+#!/usr/bin/env python
+"""Session object for building, serializing, sending, and receiving messages in
+IPython. The Session object supports serialization, HMAC signatures, and
+metadata on messages.
+
+Also defined here are utilities for working with Sessions:
+* A SessionFactory to be used as a base class for configurables that work with
+Sessions.
+* A Message object for convenience that allows attribute-access to the msg dict.
+
+Authors:
+
+* Min RK
+* Brian Granger
+* Fernando Perez
+"""
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2010-2011  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import hmac
+import logging
 import os
-import uuid
 import pprint
+import uuid
+from datetime import datetime
+
+try:
+    import cPickle
+    pickle = cPickle
+except:
+    cPickle = None
+    import pickle
 
 import zmq
+from zmq.utils import jsonapi
+from zmq.eventloop.ioloop import IOLoop
+from zmq.eventloop.zmqstream import ZMQStream
+
+from IPython.config.configurable import Configurable, LoggingConfigurable
+from IPython.utils.importstring import import_item
+from IPython.utils.jsonutil import extract_dates, squash_dates, date_default
+from IPython.utils.traitlets import CStr, Unicode, Bool, Any, Instance, Set
+
+#-----------------------------------------------------------------------------
+# utility functions
+#-----------------------------------------------------------------------------
+
+def squash_unicode(obj):
+    """coerce unicode back to bytestrings."""
+    if isinstance(obj,dict):
+        for key in obj.keys():
+            obj[key] = squash_unicode(obj[key])
+            if isinstance(key, unicode):
+                obj[squash_unicode(key)] = obj.pop(key)
+    elif isinstance(obj, list):
+        for i,v in enumerate(obj):
+            obj[i] = squash_unicode(v)
+    elif isinstance(obj, unicode):
+        obj = obj.encode('utf8')
+    return obj
+
+#-----------------------------------------------------------------------------
+# globals and defaults
+#-----------------------------------------------------------------------------
+key = 'on_unknown' if jsonapi.jsonmod.__name__ == 'jsonlib' else 'default'
+json_packer = lambda obj: jsonapi.dumps(obj, **{key:date_default})
+json_unpacker = lambda s: extract_dates(jsonapi.loads(s))
+
+pickle_packer = lambda o: pickle.dumps(o,-1)
+pickle_unpacker = pickle.loads
+
+default_packer = json_packer
+default_unpacker = json_unpacker
 
-from zmq.utils import jsonapi as json
+
+DELIM="<IDS|MSG>"
+
+#-----------------------------------------------------------------------------
+# Classes
+#-----------------------------------------------------------------------------
+
+class SessionFactory(LoggingConfigurable):
+    """The Base class for configurables that have a Session, Context, logger,
+    and IOLoop.
+    """
+    
+    logname = Unicode('')
+    def _logname_changed(self, name, old, new):
+        self.log = logging.getLogger(new)
+    
+    # not configurable:
+    context = Instance('zmq.Context')
+    def _context_default(self):
+        return zmq.Context.instance()
+    
+    session = Instance('IPython.zmq.session.Session')
+    
+    loop = Instance('zmq.eventloop.ioloop.IOLoop', allow_none=False)
+    def _loop_default(self):
+        return IOLoop.instance()
+    
+    def __init__(self, **kwargs):
+        super(SessionFactory, self).__init__(**kwargs)
+        
+        if self.session is None:
+            # construct the session
+            self.session = Session(**kwargs)
+    
 
 class Message(object):
     """A simple message object that maps dict keys to attributes.
@@ -14,7 +123,7 b' class Message(object):'
     
     def __init__(self, msg_dict):
         dct = self.__dict__
-        for k, v in msg_dict.iteritems():
+        for k, v in dict(msg_dict).iteritems():
             if isinstance(v, dict):
                 v = Message(v)
             dct[k] = v
@@ -36,13 +145,9 b' class Message(object):'
         return self.__dict__[k]
 
 
-def msg_header(msg_id, username, session):
-    return {
-        'msg_id' : msg_id,
-        'username' : username,
-        'session' : session
-    }
-
+def msg_header(msg_id, msg_type, username, session):
+    date = datetime.now()
+    return locals()
 
 def extract_header(msg_or_header):
     """Given a message or header, return the header."""
@@ -63,109 +168,446 b' def extract_header(msg_or_header):'
         h = dict(h)
     return h
 
+class Session(Configurable):
+    """Object for handling serialization and sending of messages.
+    
+    The Session object handles building messages and sending them
+    with ZMQ sockets or ZMQStream objects.  Objects can communicate with each
+    other over the network via Session objects, and only need to work with the
+    dict-based IPython message spec. The Session will handle 
+    serialization/deserialization, security, and metadata.
+    
+    Sessions support configurable serialiization via packer/unpacker traits,
+    and signing with HMAC digests via the key/keyfile traits.
+    
+    Parameters
+    ----------
+    
+    debug : bool
+        whether to trigger extra debugging statements
+    packer/unpacker : str : 'json', 'pickle' or import_string
+        importstrings for methods to serialize message parts.  If just
+        'json' or 'pickle', predefined JSON and pickle packers will be used.
+        Otherwise, the entire importstring must be used.
+        
+        The functions must accept at least valid JSON input, and output *bytes*.
+        
+        For example, to use msgpack:
+        packer = 'msgpack.packb', unpacker='msgpack.unpackb'
+    pack/unpack : callables
+        You can also set the pack/unpack callables for serialization directly.
+    session : bytes
+        the ID of this Session object.  The default is to generate a new UUID.
+    username : unicode
+        username added to message headers.  The default is to ask the OS.
+    key : bytes
+        The key used to initialize an HMAC signature.  If unset, messages
+        will not be signed or checked.
+    keyfile : filepath
+        The file containing a key.  If this is set, `key` will be initialized
+        to the contents of the file.
+    
+    """
+    
+    debug=Bool(False, config=True, help="""Debug output in the Session""")
+    
+    packer = Unicode('json',config=True,
+            help="""The name of the packer for serializing messages.
+            Should be one of 'json', 'pickle', or an import name
+            for a custom callable serializer.""")
+    def _packer_changed(self, name, old, new):
+        if new.lower() == 'json':
+            self.pack = json_packer
+            self.unpack = json_unpacker
+        elif new.lower() == 'pickle':
+            self.pack = pickle_packer
+            self.unpack = pickle_unpacker
+        else:
+            self.pack = import_item(str(new))
 
-class Session(object):
-
-    def __init__(self, username=os.environ.get('USER','username'), session=None):
-        self.username = username
-        if session is None:
-            self.session = str(uuid.uuid4())
+    unpacker = Unicode('json', config=True,
+        help="""The name of the unpacker for unserializing messages.
+        Only used with custom functions for `packer`.""")
+    def _unpacker_changed(self, name, old, new):
+        if new.lower() == 'json':
+            self.pack = json_packer
+            self.unpack = json_unpacker
+        elif new.lower() == 'pickle':
+            self.pack = pickle_packer
+            self.unpack = pickle_unpacker
         else:
-            self.session = session
-        self.msg_id = 0
+            self.unpack = import_item(str(new))
+        
+    session = CStr('', config=True,
+        help="""The UUID identifying this session.""")
+    def _session_default(self):
+        return bytes(uuid.uuid4())
+    
+    username = Unicode(os.environ.get('USER','username'), config=True,
+        help="""Username for the Session. Default is your system username.""")
+    
+    # message signature related traits:
+    key = CStr('', config=True,
+        help="""execution key, for extra authentication.""")
+    def _key_changed(self, name, old, new):
+        if new:
+            self.auth = hmac.HMAC(new)
+        else:
+            self.auth = None
+    auth = Instance(hmac.HMAC)
+    digest_history = Set()
+    
+    keyfile = Unicode('', config=True,
+        help="""path to file containing execution key.""")
+    def _keyfile_changed(self, name, old, new):
+        with open(new, 'rb') as f:
+            self.key = f.read().strip()
 
-    def msg_header(self):
-        h = msg_header(self.msg_id, self.username, self.session)
-        self.msg_id += 1
-        return h
+    pack = Any(default_packer) # the actual packer function
+    def _pack_changed(self, name, old, new):
+        if not callable(new):
+            raise TypeError("packer must be callable, not %s"%type(new))
+    
+    unpack = Any(default_unpacker) # the actual packer function
+    def _unpack_changed(self, name, old, new):
+        # unpacker is not checked - it is assumed to be 
+        if not callable(new):
+            raise TypeError("unpacker must be callable, not %s"%type(new))
 
-    def msg(self, msg_type, content=None, parent=None):
-        """Construct a standard-form message, with a given type, content, and parent.
+    def __init__(self, **kwargs):
+        """create a Session object
         
-        NOT to be called directly.
+        Parameters
+        ----------
+
+        debug : bool
+            whether to trigger extra debugging statements
+        packer/unpacker : str : 'json', 'pickle' or import_string
+            importstrings for methods to serialize message parts.  If just
+            'json' or 'pickle', predefined JSON and pickle packers will be used.
+            Otherwise, the entire importstring must be used.
+
+            The functions must accept at least valid JSON input, and output
+            *bytes*.
+
+            For example, to use msgpack:
+            packer = 'msgpack.packb', unpacker='msgpack.unpackb'
+        pack/unpack : callables
+            You can also set the pack/unpack callables for serialization
+            directly.
+        session : bytes
+            the ID of this Session object.  The default is to generate a new 
+            UUID.
+        username : unicode
+            username added to message headers.  The default is to ask the OS.
+        key : bytes
+            The key used to initialize an HMAC signature.  If unset, messages
+            will not be signed or checked.
+        keyfile : filepath
+            The file containing a key.  If this is set, `key` will be 
+            initialized to the contents of the file.
         """
+        super(Session, self).__init__(**kwargs)
+        self._check_packers()
+        self.none = self.pack({})
+
+    @property
+    def msg_id(self):
+        """always return new uuid"""
+        return str(uuid.uuid4())
+
+    def _check_packers(self):
+        """check packers for binary data and datetime support."""
+        pack = self.pack
+        unpack = self.unpack
+        
+        # check simple serialization
+        msg = dict(a=[1,'hi'])
+        try:
+            packed = pack(msg)
+        except Exception:
+            raise ValueError("packer could not serialize a simple message")
+        
+        # ensure packed message is bytes
+        if not isinstance(packed, bytes):
+            raise ValueError("message packed to %r, but bytes are required"%type(packed))
+        
+        # check that unpack is pack's inverse
+        try:
+            unpacked = unpack(packed)
+        except Exception:
+            raise ValueError("unpacker could not handle the packer's output")
+        
+        # check datetime support
+        msg = dict(t=datetime.now())
+        try:
+            unpacked = unpack(pack(msg))
+        except Exception:
+            self.pack = lambda o: pack(squash_dates(o))
+            self.unpack = lambda s: extract_dates(unpack(s))
+    
+    def msg_header(self, msg_type):
+        return msg_header(self.msg_id, msg_type, self.username, self.session)
+
+    def msg(self, msg_type, content=None, parent=None, subheader=None):
         msg = {}
-        msg['header'] = self.msg_header()
+        msg['header'] = self.msg_header(msg_type)
+        msg['msg_id'] = msg['header']['msg_id']
         msg['parent_header'] = {} if parent is None else extract_header(parent)
         msg['msg_type'] = msg_type
         msg['content'] = {} if content is None else content
+        sub = {} if subheader is None else subheader
+        msg['header'].update(sub)
         return msg
 
-    def send(self, socket, msg_or_type, content=None, parent=None, ident=None):
-        """send a message via a socket, using a uniform message pattern.
-        
-        Parameters
-        ----------
-        socket : zmq.Socket
-            The socket on which to send.
-        msg_or_type : Message/dict or str
-            if str : then a new message will be constructed from content,parent
-            if Message/dict : then content and parent are ignored, and the message
-                is sent.  This is only for use when sending a Message for a second time.
-        content : dict, optional
-            The contents of the message
-        parent : dict, optional
-            The parent header, or parent message, of this message
-        ident : bytes, optional
-            The zmq.IDENTITY prefix of the destination.
-            Only for use on certain socket types.
+    def sign(self, msg):
+        """Sign a message with HMAC digest. If no auth, return b''."""
+        if self.auth is None:
+            return b''
+        h = self.auth.copy()
+        for m in msg:
+            h.update(m)
+        return h.hexdigest()
+    
+    def serialize(self, msg, ident=None):
+        """Serialize the message components to bytes.
         
         Returns
         -------
-        msg : dict
-            The message, as constructed by self.msg(msg_type,content,parent)
+        
+        list of bytes objects
+        
         """
-        if isinstance(msg_or_type, (Message, dict)):
-            msg = dict(msg_or_type)
+        content = msg.get('content', {})
+        if content is None:
+            content = self.none
+        elif isinstance(content, dict):
+            content = self.pack(content)
+        elif isinstance(content, bytes):
+            # content is already packed, as in a relayed message
+            pass
+        elif isinstance(content, unicode):
+            # should be bytes, but JSON often spits out unicode
+            content = content.encode('utf8')
         else:
-            msg = self.msg(msg_or_type, content, parent)
-        if ident is not None:
-            socket.send(ident, zmq.SNDMORE)
-        socket.send_json(msg)
-        return msg
+            raise TypeError("Content incorrect type: %s"%type(content))
+        
+        real_message = [self.pack(msg['header']), 
+                        self.pack(msg['parent_header']), 
+                        content
+        ]
+        
+        to_send = []
+
+        if isinstance(ident, list):
+            # accept list of idents
+            to_send.extend(ident)
+        elif ident is not None:
+            to_send.append(ident)
+        to_send.append(DELIM)
+        
+        signature = self.sign(real_message)
+        to_send.append(signature)
+        
+        to_send.extend(real_message)
 
-    def recv(self, socket, mode=zmq.NOBLOCK):
-        """recv a message on a socket.
+        return to_send
         
-        Receive an optionally identity-prefixed message, as sent via session.send().
+    def send(self, stream, msg_or_type, content=None, parent=None, ident=None, 
+                                    buffers=None, subheader=None, track=False):
+        """Build and send a message via stream or socket.
         
         Parameters
         ----------
         
-        socket : zmq.Socket
-            The socket on which to recv a message.
-        mode : int, optional
-            the mode flag passed to socket.recv
-            default: zmq.NOBLOCK
+        stream : zmq.Socket or ZMQStream
+            the socket-like object used to send the data
+        msg_or_type : str or Message/dict
+            Normally, msg_or_type will be a msg_type unless a message is being 
+            sent more than once.
+        
+        content : dict or None
+            the content of the message (ignored if msg_or_type is a message)
+        parent : Message or dict or None
+            the parent or parent header describing the parent of this message
+        ident : bytes or list of bytes
+            the zmq.IDENTITY routing path
+        subheader : dict or None
+            extra header keys for this message's header
+        buffers : list or None
+            the already-serialized buffers to be appended to the message
+        track : bool
+            whether to track.  Only for use with Sockets, 
+            because ZMQStream objects cannot track messages.
         
         Returns
         -------
-        (ident,msg) : tuple
-            always length 2. If no message received, then return is (None,None)
-        ident : bytes or None
-                the identity prefix is there was one, None otherwise.
-        msg : dict or None
-                The actual message.  If mode==zmq.NOBLOCK and no message was waiting,
-                it will be None.
+        msg : message dict
+            the constructed message
+        (msg,tracker) : (message dict, MessageTracker)
+            if track=True, then a 2-tuple will be returned, 
+            the first element being the constructed
+            message, and the second being the MessageTracker
+            
         """
+
+        if not isinstance(stream, (zmq.Socket, ZMQStream)):
+            raise TypeError("stream must be Socket or ZMQStream, not %r"%type(stream))
+        elif track and isinstance(stream, ZMQStream):
+            raise TypeError("ZMQStream cannot track messages")
+        
+        if isinstance(msg_or_type, (Message, dict)):
+            # we got a Message, not a msg_type
+            # don't build a new Message
+            msg = msg_or_type
+        else:
+            msg = self.msg(msg_or_type, content, parent, subheader)
+        
+        buffers = [] if buffers is None else buffers
+        to_send = self.serialize(msg, ident)
+        flag = 0
+        if buffers:
+            flag = zmq.SNDMORE
+            _track = False
+        else:
+            _track=track
+        if track:
+            tracker = stream.send_multipart(to_send, flag, copy=False, track=_track)
+        else:
+            tracker = stream.send_multipart(to_send, flag, copy=False)
+        for b in buffers[:-1]:
+            stream.send(b, flag, copy=False)
+        if buffers:
+            if track:
+                tracker = stream.send(buffers[-1], copy=False, track=track)
+            else:
+                tracker = stream.send(buffers[-1], copy=False)
+                
+        # omsg = Message(msg)
+        if self.debug:
+            pprint.pprint(msg)
+            pprint.pprint(to_send)
+            pprint.pprint(buffers)
+        
+        msg['tracker'] = tracker
+        
+        return msg
+    
+    def send_raw(self, stream, msg, flags=0, copy=True, ident=None):
+        """Send a raw message via ident path.
+        
+        Parameters
+        ----------
+        msg : list of sendable buffers"""
+        to_send = []
+        if isinstance(ident, bytes):
+            ident = [ident]
+        if ident is not None:
+            to_send.extend(ident)
+            
+        to_send.append(DELIM)
+        to_send.append(self.sign(msg))
+        to_send.extend(msg)
+        stream.send_multipart(msg, flags, copy=copy)
+    
+    def recv(self, socket, mode=zmq.NOBLOCK, content=True, copy=True):
+        """receives and unpacks a message
+        returns [idents], msg"""
+        if isinstance(socket, ZMQStream):
+            socket = socket.socket
         try:
             msg = socket.recv_multipart(mode)
-        except zmq.ZMQError, e:
+        except zmq.ZMQError as e:
             if e.errno == zmq.EAGAIN:
                 # We can convert EAGAIN to None as we know in this case
-                # recv_json won't return None.
+                # recv_multipart won't return None.
                 return None,None
             else:
                 raise
-        if len(msg) == 1:
-            ident=None
-            msg = msg[0]
-        elif len(msg) == 2:
-            ident, msg = msg
+        # split multipart message into identity list and message dict
+        # invalid large messages can cause very expensive string comparisons
+        idents, msg = self.feed_identities(msg, copy)
+        try:
+            return idents, self.unpack_message(msg, content=content, copy=copy)
+        except Exception as e:
+            print (idents, msg)
+            # TODO: handle it
+            raise e
+    
+    def feed_identities(self, msg, copy=True):
+        """feed until DELIM is reached, then return the prefix as idents and
+        remainder as msg. This is easily broken by setting an IDENT to DELIM,
+        but that would be silly.
+        
+        Parameters
+        ----------
+        msg : a list of Message or bytes objects
+            the message to be split
+        copy : bool
+            flag determining whether the arguments are bytes or Messages
+        
+        Returns
+        -------
+        (idents,msg) : two lists
+            idents will always be a list of bytes - the indentity prefix
+            msg will be a list of bytes or Messages, unchanged from input
+            msg should be unpackable via self.unpack_message at this point.
+        """
+        if copy:
+            idx = msg.index(DELIM)
+            return msg[:idx], msg[idx+1:]
         else:
-            raise ValueError("Got message with length > 2, which is invalid")
+            failed = True
+            for idx,m in enumerate(msg):
+                if m.bytes == DELIM:
+                    failed = False
+                    break
+            if failed:
+                raise ValueError("DELIM not in msg")
+            idents, msg = msg[:idx], msg[idx+1:]
+            return [m.bytes for m in idents], msg
+    
+    def unpack_message(self, msg, content=True, copy=True):
+        """Return a message object from the format
+        sent by self.send.
+        
+        Parameters:
+        -----------
         
-        return ident, json.loads(msg)
+        content : bool (True)
+            whether to unpack the content dict (True), 
+            or leave it serialized (False)
+        
+        copy : bool (True)
+            whether to return the bytes (True), 
+            or the non-copying Message object in each place (False)
+        
+        """
+        minlen = 4
+        message = {}
+        if not copy:
+            for i in range(minlen):
+                msg[i] = msg[i].bytes
+        if self.auth is not None:
+            signature = msg[0]
+            if signature in self.digest_history:
+                raise ValueError("Duplicate Signature: %r"%signature)
+            self.digest_history.add(signature)
+            check = self.sign(msg[1:4])
+            if not signature == check:
+                raise ValueError("Invalid Signature: %r"%signature)
+        if not len(msg) >= minlen:
+            raise TypeError("malformed message, must have at least %i elements"%minlen)
+        message['header'] = self.unpack(msg[1])
+        message['msg_type'] = message['header']['msg_type']
+        message['parent_header'] = self.unpack(msg[2])
+        if content:
+            message['content'] = self.unpack(msg[3])
+        else:
+            message['content'] = msg[3]
+        
+        message['buffers'] = msg[4:]
+        return message
 
 def test_msg2obj():
     am = dict(x=1)
@@ -182,3 +624,4 b' def test_msg2obj():'
     am2 = dict(ao)
     assert am['x'] == am2['x']
     assert am['y']['z'] == am2['y']['z']
+

IPython/zmq/tests/test_message_spec.py

@@ -35,6 +35,6 b' def teardown():'
 # Actual tests
 
 def test_execute():
-    KM.xreq_channel.execute(code='x=1')
-    KM.xreq_channel.execute(code='print 1')
+    KM.shell_channel.execute(code='x=1')
+    KM.shell_channel.execute(code='print 1')
     

IPython/zmq/tests/test_session.py

@@ -17,14 +17,14 b' import zmq'
 
 from zmq.tests import BaseZMQTestCase
 from zmq.eventloop.zmqstream import ZMQStream
-# from IPython.zmq.tests import SessionTestCase
-from IPython.parallel import streamsession as ss
+
+from IPython.zmq import session as ss
 
 class SessionTestCase(BaseZMQTestCase):
     
     def setUp(self):
         BaseZMQTestCase.setUp(self)
-        self.session = ss.StreamSession()
+        self.session = ss.Session()
 
 class TestSession(SessionTestCase):
     
@@ -42,19 +42,19 b' class TestSession(SessionTestCase):'
             
     
     def test_args(self):
-        """initialization arguments for StreamSession"""
+        """initialization arguments for Session"""
         s = self.session
         self.assertTrue(s.pack is ss.default_packer)
         self.assertTrue(s.unpack is ss.default_unpacker)
         self.assertEquals(s.username, os.environ.get('USER', 'username'))
         
-        s = ss.StreamSession(username=None)
+        s = ss.Session()
         self.assertEquals(s.username, os.environ.get('USER', 'username'))
         
-        self.assertRaises(TypeError, ss.StreamSession, packer='hi')
-        self.assertRaises(TypeError, ss.StreamSession, unpacker='hi')
+        self.assertRaises(TypeError, ss.Session, pack='hi')
+        self.assertRaises(TypeError, ss.Session, unpack='hi')
         u = str(uuid.uuid4())
-        s = ss.StreamSession(username='carrot', session=u)
+        s = ss.Session(username='carrot', session=u)
         self.assertEquals(s.session, u)
         self.assertEquals(s.username, 'carrot')
         

docs/examples/core/appconfig.py

@@ -56,7 +56,7 b' class Bar(Configurable):'
 
 class MyApp(Application):
 
-    app_name = Unicode(u'myapp')
+    name = Unicode(u'myapp')
     running = Bool(False, config=True,
                    help="Is the app running?")
     classes = List([Bar, Foo])
@@ -79,17 +79,22 b' class MyApp(Application):'
         # Pass config to other classes for them to inherit the config.
         self.bar = Bar(config=self.config)
 
+    def initialize(self, argv=None):
+        self.parse_command_line(argv)
+        if self.config_file:
+            self.load_config_file(self.config_file)
+        self.init_foo()
+        self.init_bar()
+    
+    def start(self):
+        print "app.config:"
+        print self.config
 
 
 def main():
     app = MyApp()
-    app.parse_command_line()
-    if app.config_file:
-        app.load_config_file(app.config_file)
-    app.init_foo()
-    app.init_bar()
-    print "app.config:"
-    print app.config
+    app.initialize()
+    app.start()
 
 
 if __name__ == "__main__":

docs/source/config/extensions.txt

@@ -46,7 +46,7 b' There are two ways you can tell IPython to use your extension:'
 To load an extension called :file:`myextension.py` add the following logic
 to your configuration file::
 
-    c.Global.extensions = [
+    c.InteractiveShellApp.extensions = [
         'myextension'
     ]
 

docs/source/config/ipython.txt

@@ -15,63 +15,74 b' can configure the application. A sample is provided in'
 :mod:`IPython.config.default.ipython_config`.  Simply copy this file to your
 :ref:`IPython directory <ipython_dir>` to start using it.
 
-Most configuration attributes that this file accepts are associated with
-classes that are subclasses of :class:`~IPython.core.component.Component`.
+Most configuration attributes that this file accepts are associated with classes
+that are subclasses of :class:`~IPython.config.configurable.Configurable`.  
 
-A few configuration attributes are not associated with a particular 
-:class:`~IPython.core.component.Component` subclass.  These are application
-wide configuration attributes and are stored in the ``Global`` 
-sub-configuration section.  We begin with a description of these 
-attributes.
+Applications themselves are Configurable as well, so we will start with some
+application-level config.
 
-Global configuration
-====================
+Application-level configuration
+===============================
 
 Assuming that your configuration file has the following at the top::
 
     c = get_config()
 
-the following attributes can be set in the ``Global`` section.
+the following attributes are set application-wide:
 
-:attr:`c.Global.display_banner`
+terminal IPython-only flags:
+
+:attr:`c.TerminalIPythonApp.display_banner`
     A boolean that determined if the banner is printer when :command:`ipython`
     is started.
 
-:attr:`c.Global.classic`
+:attr:`c.TerminalIPythonApp.classic`
     A boolean that determines if IPython starts in "classic" mode.  In this
     mode, the prompts and everything mimic that of the normal :command:`python`
     shell
 
-:attr:`c.Global.nosep`
+:attr:`c.TerminalIPythonApp.nosep`
     A boolean that determines if there should be no blank lines between
     prompts.
 
-:attr:`c.Global.log_level`
+:attr:`c.Application.log_level`
     An integer that sets the detail of the logging level during the startup
     of :command:`ipython`.  The default is 30 and the possible values are
     (0, 10, 20, 30, 40, 50).  Higher is quieter and lower is more verbose.
+    This can also be set by the name of the logging level, e.g. INFO=20,
+    WARN=30.
+
+Some options, such as extensions and startup code, can be set for any
+application that starts an
+:class:`~IPython.core.interactiveshell.InteractiveShell`. These apps are
+subclasses of :class:`~IPython.core.shellapp.InteractiveShellApp`. Since
+subclasses inherit configuration, setting a trait of
+:attr:`c.InteractiveShellApp` will affect all IPython applications, but if you
+want terminal IPython and the QtConsole to have different values, you can set
+them via :attr:`c.TerminalIPythonApp` and :attr:`c.IPKernelApp` respectively.
+
 
-:attr:`c.Global.extensions`
+:attr:`c.InteractiveShellApp.extensions`
     A list of strings, each of which is an importable IPython extension. An
     IPython extension is a regular Python module or package that has a
     :func:`load_ipython_extension(ip)` method. This method gets called when
     the extension is loaded with the currently running
-    :class:`~IPython.core.iplib.InteractiveShell` as its only argument. You
-    can put your extensions anywhere they can be imported but we add the
-    :file:`extensions` subdirectory of the ipython directory to ``sys.path``
-    during extension loading, so you can put them there as well. Extensions
-    are not executed in the user's interactive namespace and they must be pure
-    Python code. Extensions are the recommended way of customizing
+    :class:`~IPython.core.interactiveshell.InteractiveShell` as its only
+    argument. You can put your extensions anywhere they can be imported but we
+    add the :file:`extensions` subdirectory of the ipython directory to
+    ``sys.path`` during extension loading, so you can put them there as well.
+    Extensions are not executed in the user's interactive namespace and they
+    must be pure Python code. Extensions are the recommended way of customizing
     :command:`ipython`. Extensions can provide an
     :func:`unload_ipython_extension` that will be called when the extension is
     unloaded.
 
-:attr:`c.Global.exec_lines`
+:attr:`c.InteractiveShellApp.exec_lines`
     A list of strings, each of which is Python code that is run in the user's
     namespace after IPython start. These lines can contain full IPython syntax
     with magics, etc.
 
-:attr:`c.Global.exec_files`
+:attr:`c.InteractiveShellApp.exec_files`
     A list of strings, each of which is the full pathname of a ``.py`` or
     ``.ipy`` file that will be executed as IPython starts. These files are run
     in IPython in the user's namespace. Files with a ``.py`` extension need to
@@ -85,7 +96,7 b' Classes that can be configured'
 The following classes can also be configured in the configuration file for
 :command:`ipython`:
 
-* :class:`~IPython.core.iplib.InteractiveShell`
+* :class:`~IPython.core.interactiveshell.InteractiveShell`
 
 * :class:`~IPython.core.prefilter.PrefilterManager`
 
@@ -105,16 +116,16 b' attributes::'
     # sample ipython_config.py
     c = get_config()
 
-    c.Global.display_banner = True
-    c.Global.log_level = 20
-    c.Global.extensions = [
+    c.IPythonTerminalApp.display_banner = True
+    c.InteractiveShellApp.log_level = 20
+    c.InteractiveShellApp.extensions = [
         'myextension'
     ]
-    c.Global.exec_lines = [
+    c.InteractiveShellApp.exec_lines = [
         'import numpy',
         'import scipy'
     ]
-    c.Global.exec_files = [
+    c.InteractiveShellApp.exec_files = [
         'mycode.py',
         'fancy.ipy'
     ]

docs/source/config/overview.txt

@@ -18,8 +18,8 b' met our requirements.'
     your old :file:`ipythonrc` or :file:`ipy_user_conf.py` configuration files
     to the new system.  Read on for information on how to do this.
 
-The discussion that follows is focused on teaching user's how to configure
-IPython to their liking.  Developer's who want to know more about how they
+The discussion that follows is focused on teaching users how to configure
+IPython to their liking.  Developers who want to know more about how they
 can enable their objects to take advantage of the configuration system
 should consult our :ref:`developer guide <developer_guide>`
 
@@ -37,15 +37,19 b' Configuration object: :class:`~IPython.config.loader.Config`'
     are smart. They know how to merge themselves with other configuration
     objects and they automatically create sub-configuration objects.
 
-Application: :class:`~IPython.core.application.Application`
+Application: :class:`~IPython.config.application.Application`
     An application is a process that does a specific job. The most obvious
     application is the :command:`ipython` command line program. Each
-    application reads a *single* configuration file and command line options
+    application reads *one or more* configuration files and a single set of
+    command line options
     and then produces a master configuration object for the application. This
     configuration object is then passed to the configurable objects that the
     application creates. These configurable objects implement the actual logic
     of the application and know how to configure themselves given the
     configuration object.
+    
+    Applications always have a `log` attribute that is a configured Logger.
+    This allows centralized logging configuration per-application.
 
 Component: :class:`~IPython.config.configurable.Configurable`
     A configurable is a regular Python class that serves as a base class for
@@ -64,6 +68,25 b' Component: :class:`~IPython.config.configurable.Configurable`'
     these subclasses has its own configuration information that controls how
     instances are created.
 
+Singletons: :class:`~IPython.config.configurable.SingletonConfigurable`
+    Any object for which there is a single canonical instance. These are
+    just like Configurables, except they have a class method 
+    :meth:`~IPython.config.configurable.SingletonConfigurable.instance`,
+    that returns the current active instance (or creates one if it
+    does not exist).  Examples of singletons include
+    :class:`~IPython.config.application.Application`s and
+    :class:`~IPython.core.interactiveshell.InteractiveShell`.  This lets
+    objects easily connect to the current running Application without passing
+    objects around everywhere.  For instance, to get the current running 
+    Application instance, simply do: ``app = Application.instance()``.
+
+
+.. note::
+
+    Singletons are not strictly enforced - you can have many instances
+    of a given singleton class, but the :meth:`instance` method will always
+    return the same one.
+
 Having described these main concepts, we can now state the main idea in our
 configuration system: *"configuration" allows the default values of class
 attributes to be controlled on a class by class basis*. Thus all instances of
@@ -106,7 +129,7 b' subclass::'
     from IPython.utils.traitlets import Int, Float, Str, Bool
     
     class MyClass(Configurable):
-        name = Str('defaultname', config=True)
+        name = Unicode(u'defaultname', config=True)
         ranking = Int(0, config=True)
         value = Float(99.0)
         # The rest of the class implementation would go here..
@@ -197,6 +220,15 b' search path for sub-configuration files is inherited from that of the parent.'
 Thus, you can typically put the two in the same directory and everything will
 just work.
 
+You can also load configuration files by profile, for instance:
+
+.. sourcecode:: python
+
+    load_subconfig('ipython_config.py', profile='default')
+
+to inherit your default configuration as a starting point.
+
+
 Class based configuration inheritance
 =====================================
 
@@ -241,11 +273,12 b' This class hierarchy and configuration file accomplishes the following:'
 Configuration file location
 ===========================
 
-So where should you put your configuration files?  By default, all IPython
-applications look in the so called "IPython directory".  The location of
-this directory is determined by the following algorithm:
+So where should you put your configuration files? IPython uses "profiles" for
+configuration, and by default, all profiles will be stored in the so called
+"IPython directory". The location of this directory is determined by the
+following algorithm:
 
-* If the ``--ipython-dir`` command line flag is given, its value is used.
+* If the ``ipython_dir`` command line flag is given, its value is used.
 
 * If not, the value returned by :func:`IPython.utils.path.get_ipython_dir`
   is used. This function will first look at the :envvar:`IPYTHON_DIR`
@@ -263,45 +296,149 b' For most users, the default value will simply be something like'
 :file:`$HOME/.config/ipython` on Linux, or :file:`$HOME/.ipython`
 elsewhere.
 
-Once the location of the IPython directory has been determined, you need to
-know what filename to use for the configuration file. The basic idea is that
-each application has its own default configuration filename. The default named
-used by the :command:`ipython` command line program is
-:file:`ipython_config.py`.  This value can be overriden by the ``-config_file``
-command line flag.  A sample :file:`ipython_config.py` file can be found
-in :mod:`IPython.config.default.ipython_config.py`.  Simple copy it to your
-IPython directory to begin using it.
+Once the location of the IPython directory has been determined, you need to know
+which profile you are using. For users with a single configuration, this will
+simply be 'default', and will be located in 
+:file:`<IPYTHON_DIR>/profile_default`.
+
+The next thing you need to know is what to call your configuration file. The
+basic idea is that each application has its own default configuration filename.
+The default named used by the :command:`ipython` command line program is
+:file:`ipython_config.py`, and *all* IPython applications will use this file.
+Other applications, such as the parallel :command:`ipcluster` scripts or the
+QtConsole will load their own config files *after* :file:`ipython_config.py`. To
+load a particular configuration file instead of the default, the name can be
+overridden by the ``config_file`` command line flag.
+
+To generate the default configuration files, do::
+
+    $> ipython profile create
+
+and you will have a default :file:`ipython_config.py` in your IPython directory
+under :file:`profile_default`. If you want the default config files for the
+:mod:`IPython.parallel` applications, add ``--parallel`` to the end of the
+command-line args.
 
 .. _Profiles:
 
 Profiles
 ========
 
-A profile is simply a configuration file that follows a simple naming
-convention and can be loaded using a simplified syntax.  The idea is
-that users often want to maintain a set of configuration files for different
-purposes:  one for doing numerical computing with NumPy and SciPy and
-another for doing symbolic computing with SymPy.  Profiles make it easy
-to keep a separate configuration file for each of these purposes.
+A profile is a directory containing configuration and runtime files, such as
+logs, connection info for the parallel apps, and your IPython command history.
+
+The idea is that users often want to maintain a set of configuration files for
+different purposes: one for doing numerical computing with NumPy and SciPy and
+another for doing symbolic computing with SymPy. Profiles make it easy to keep a
+separate configuration files, logs, and histories for each of these purposes.
 
 Let's start by showing how a profile is used:
 
 .. code-block:: bash
 
-    $ ipython -p sympy
+    $ ipython profile=sympy
+
+This tells the :command:`ipython` command line program to get its configuration
+from the "sympy" profile. The file names for various profiles do not change. The
+only difference is that profiles are named in a special way. In the case above,
+the "sympy" profile means looking for :file:`ipython_config.py` in :file:`<IPYTHON_DIR>/profile_sympy`.
+
+The general pattern is this: simply create a new profile with:
+
+.. code-block:: bash
+
+    ipython profile create <name>
+
+which adds a directory called ``profile_<name>`` to your IPython directory. Then
+you can load this profile by adding ``profile=<name>`` to your command line
+options. Profiles are supported by all IPython applications.
+
+IPython ships with some sample profiles in :file:`IPython/config/profile`. If
+you create profiles with the name of one of our shipped profiles, these config
+files will be copied over instead of starting with the automatically generated
+config files.
+
+.. _commandline:
+
+Command-line arguments
+======================
+
+IPython exposes *all* configurable options on the command-line. The command-line
+arguments are generated from the Configurable traits of the classes associated
+with a given Application.  Configuring IPython from the command-line may look
+very similar to an IPython config file
+
+IPython applications use a parser called
+:class:`~IPython.config.loader.KeyValueLoader` to load values into a Config
+object.  Values are assigned in much the same way as in a config file:
+
+.. code-block:: bash
+
+    $> ipython InteractiveShell.use_readline=False BaseIPythonApplication.profile='myprofile'
+
+Is the same as adding:
+
+.. sourcecode:: python
+
+    c.InteractiveShell.use_readline=False
+    c.BaseIPythonApplication.profile='myprofile'
+
+to your config file. Key/Value arguments *always* take a value, separated by '='
+and no spaces.
+
+Aliases
+-------
+
+For convenience, applications have a mapping of commonly
+used traits, so you don't have to specify the whole class name. For these **aliases**, the class need not be specified:
+
+.. code-block:: bash
+
+    $> ipython profile='myprofile'
+    # is equivalent to
+    $> ipython BaseIPythonApplication.profile='myprofile'
+
+Flags
+-----
+
+Applications can also be passed **flags**. Flags are options that take no
+arguments, and are always prefixed with ``--``. They are simply wrappers for
+setting one or more configurables with predefined values, often True/False.
+
+For instance:
+
+.. code-block:: bash
+
+    $> ipcontroller --debug
+    # is equivalent to
+    $> ipcontroller Application.log_level=DEBUG
+    # and 
+    $> ipython --pylab
+    # is equivalent to
+    $> ipython pylab=auto
+
+Subcommands
+-----------
+
+
+Some IPython applications have **subcommands**. Subcommands are modeled after
+:command:`git`, and are called with the form :command:`command subcommand
+[...args]`.  Currently, the QtConsole is a subcommand of terminal IPython:
+
+.. code-block:: bash
+
+    $> ipython qtconsole profile=myprofile
+
+and :command:`ipcluster` is simply a wrapper for its various subcommands (start,
+stop, engines).
+
+.. code-block:: bash
+
+    $> ipcluster start profile=myprofile n=4
 
-This tells the :command:`ipython` command line program to get its
-configuration from the "sympy" profile. The search path for profiles is the
-same as that of regular configuration files. The only difference is that
-profiles are named in a special way. In the case above, the "sympy" profile
-would need to have the name :file:`ipython_config_sympy.py`.
 
-The general pattern is this: simply add ``_profilename`` to the end of the
-normal configuration file name. Then load the profile by adding ``-p
-profilename`` to your command line options.
+To see a list of the available aliases, flags, and subcommands for an IPython application, simply pass ``-h`` or ``--help``.  And to see the full list of configurable options (*very* long), pass ``--help-all``.
 
-IPython ships with some sample profiles in :mod:`IPython.config.profile`.
-Simply copy these to your IPython directory to begin using them.
 
 Design requirements
 ===================

docs/source/development/index.txt

@@ -23,4 +23,3 b" IPython developer's guide"
    ipgraph.txt
    ipython_qt.txt
    ipythonzmq.txt
-   parallelzmq.txt

docs/source/install/install.txt

@@ -303,11 +303,10 b' pyzmq'
 IPython 0.11 introduced some new functionality, including a two-process
 execution model using ZeroMQ for communication [ZeroMQ]_. The Python bindings
 to ZeroMQ are found in the pyzmq project, which is easy_install-able once you
-have ZeroMQ installed (or even if you don't).
+have ZeroMQ installed.  If you are on Python 2.6 or 2.7 on OSX, or 2.7 on Windows,
+pyzmq has eggs that include ZeroMQ itself.
 
-IPython.zmq depends on pyzmq >= 2.0.10.1, but IPython.parallel requires the more
-recent 2.1.4.  2.1.4 also has binary releases for OSX and Windows, that do not
-require prior installation of libzmq.
+IPython.zmq depends on pyzmq >= 2.1.4.
 
 Dependencies for ipython-qtconsole (new GUI)
 ============================================

docs/source/interactive/qtconsole.txt

@@ -59,16 +59,16 b' input and the drawing eventloop.'
 ****************
 
 An additional function, :func:`pastefig`, will be added to the global namespace if you
-specify the ``--pylab`` argument. This takes the active figures in matplotlib, and embeds
+specify the ``pylab`` argument. This takes the active figures in matplotlib, and embeds
 them in your document. This is especially useful for saving_ your work.
 
 .. _inline:
 
-``--pylab inline``
+``pylab=inline``
 ******************
 
 If you want to have all of your figures embedded in your session, instead of calling
-:func:`pastefig`, you can specify ``--pylab inline``, and each time you make a plot, it
+:func:`pastefig`, you can specify ``pylab=inline``, and each time you make a plot, it
 will show up in your document, as if you had called :func:`pastefig`.
 
 
@@ -89,8 +89,8 b' context menu.'
 
 .. Note:: 
 
-    Saving is only available to richtext Qt widgets, so make sure you start ipqt with the
-    ``--rich`` flag, or with ``--pylab``, which always uses a richtext widget.
+    Saving is only available to richtext Qt widgets, which are used by default, but
+    if you pass the ``--plain`` flag, saving will not be available to you.
 
 
 See these examples of :download:`png/html<figs/jn.html>` and :download:`svg/xhtml
@@ -101,23 +101,22 b' Colors and Highlighting'
 =======================
 
 Terminal IPython has always had some coloring, but never syntax highlighting. There are a
-few simple color choices, specified by the ``--colors`` flag or ``%colors`` magic:
+few simple color choices, specified by the ``colors`` flag or ``%colors`` magic:
 
 * LightBG for light backgrounds
 * Linux for dark backgrounds
 * NoColor for a simple colorless terminal
 
-The Qt widget has full support for the ``--colors`` flag, but adds new, more intuitive
-aliases for the colors (the old names still work): dark=Linux, light=LightBG, bw=NoColor.
+The Qt widget has full support for the ``colors`` flag used in the terminal shell.
 
 The Qt widget, however, has full syntax highlighting as you type, handled by the
-`pygments`_ library. The ``--style`` argument exposes access to any style by name that can
-be found by pygments, and there are several already installed. The ``--colors`` argument,
+`pygments`_ library. The ``style`` argument exposes access to any style by name that can
+be found by pygments, and there are several already installed. The ``colors`` argument,
 if unspecified, will be guessed based on the chosen style. Similarly, there are default
-styles associated with each ``--colors`` option.
+styles associated with each ``colors`` option.
 
 
-Screenshot of ``ipython-qtconsole --colors dark``, which uses the 'monokai' theme by
+Screenshot of ``ipython-qtconsole colors=linux``, which uses the 'monokai' theme by
 default:
 
 .. image:: figs/colors_dark.png
@@ -129,7 +128,7 b' default:'
     on your system.
 
 You can also pass the filename of a custom CSS stylesheet, if you want to do your own
-coloring, via the ``--stylesheet`` argument.  The default LightBG stylesheet:
+coloring, via the ``stylesheet`` argument.  The default LightBG stylesheet:
 
 .. sourcecode:: css
 
@@ -142,6 +141,14 b' coloring, via the ``--stylesheet`` argument. The default LightBG stylesheet:'
     .out-prompt { color: darkred; }
     .out-prompt-number { font-weight: bold; }
 
+Fonts
+=====
+
+The QtConsole has configurable via the ConsoleWidget. To change these, set the ``font_family``
+or ``font_size`` traits of the ConsoleWidget. For instance, to use 9pt Anonymous Pro::
+
+    $> ipython-qtconsole ConsoleWidget.font_family="Anonymous Pro" ConsoleWidget.font_size=9
+
 Process Management
 ==================
 
@@ -160,7 +167,7 b' do not have to all be qt frontends - any IPython frontend can connect and run co'
 When you start ipython-qtconsole, there will be an output line, like::
 
     To connect another client to this kernel, use:
-    -e --xreq 62109 --sub 62110 --rep 62111 --hb 62112
+    --external shell=62109 iopub=62110 stdin=62111 hb=62112
 
 Other frontends can connect to your kernel, and share in the execution. This is great for
 collaboration. The `-e` flag is for 'external'. Starting other consoles with that flag
@@ -169,9 +176,9 b' have to specify each port individually, but for now this copy-paste method is be'
 
 By default (for security reasons), the kernel only listens on localhost, so you can only
 connect multiple frontends to the kernel from your local machine. You can specify to
-listen on an external interface by specifying the ``--ip`` argument::
+listen on an external interface by specifying the ``ip`` argument::
 
-    $> ipython-qtconsole --ip 192.168.1.123
+    $> ipython-qtconsole ip=192.168.1.123
 
 If you specify the ip as 0.0.0.0, that refers to all interfaces, so any computer that can
 see yours can connect to the kernel.
@@ -208,10 +215,6 b' Regressions'
 There are some features, where the qt console lags behind the Terminal frontend. We hope
 to have these fixed by 0.11 release.
 
-    * Configuration: The Qt frontend and ZMQ kernel are not yet hooked up to the IPython
-      configuration system
-    * History Persistence: Currently the history of a GUI session does
-      not persist between sessions.
     * !cmd input: Due to our use of pexpect, we cannot pass input to subprocesses launched
       using the '!' escape. (this will not be fixed).
 

docs/source/interactive/reference.txt

@@ -21,7 +21,7 b' You start IPython with the command::'
 
 If invoked with no options, it executes all the files listed in sequence
 and drops you into the interpreter while still acknowledging any options
-you may have set in your ipythonrc file. This behavior is different from
+you may have set in your ipython_config.py. This behavior is different from
 standard Python, which when called as python -i will only execute one
 file and ignore your configuration setup.
 
@@ -41,9 +41,12 b' Special Threading Options'
 
 Previously IPython had command line options for controlling GUI event loop
 integration (-gthread, -qthread, -q4thread, -wthread, -pylab). As of IPython
-version 0.11, these have been deprecated. Please see the new ``%gui``
+version 0.11, these have been removed. Please see the new ``%gui``
 magic command or :ref:`this section <gui_support>` for details on the new
-interface.
+interface, or specify the gui at the commandline::
+
+    $ ipython gui=qt
+
 
 Regular Options
 ---------------
@@ -58,15 +61,15 b' the provided example for more details on what the options do. Options'
 given at the command line override the values set in the ipythonrc file.
 
 All options with a [no] prepended can be specified in negated form
-(-nooption instead of -option) to turn the feature off.
+(--no-option instead of --option) to turn the feature off.
 
-    -help  print a help message and exit.
+    -h, --help  print a help message and exit.
 
-    -pylab
-    Deprecated.  See :ref:`Matplotlib support <matplotlib_support>`
+    --pylab, pylab=<name>
+    See :ref:`Matplotlib support <matplotlib_support>`
     for more details.
 
-    -autocall <val>
+    autocall=<val>
 	Make IPython automatically call any callable object even if you
 	didn't type explicit parentheses. For example, 'str 43' becomes
 	'str(43)' automatically. The value can be '0' to disable the feature,
@@ -75,25 +78,25 b' All options with a [no] prepended can be specified in negated form'
 	objects are automatically called (even if no arguments are
 	present). The default is '1'.
 
-    -[no]autoindent    
+    --[no-]autoindent    
 	Turn automatic indentation on/off.
 
-    -[no]automagic 
+    --[no-]automagic 
 	make magic commands automatic (without needing their first character
 	to be %). Type %magic at the IPython prompt for more information.
 
-    -[no]autoedit_syntax 
+    --[no-]autoedit_syntax 
 	When a syntax error occurs after editing a file, automatically
 	open the file to the trouble causing line for convenient
 	fixing.
 
-    -[no]banner		Print the initial information banner (default on).
+    --[no-]banner		Print the initial information banner (default on).
 
-    -c <command>
+    c=<command>
 	execute the given command string. This is similar to the -c
 	option in the normal Python interpreter.
 
-    -cache_size, cs <n> 
+    cache_size=<n> 
 	size of the output cache (maximum number of entries to hold in
 	memory). The default is 1000, you can change it permanently in your
 	config file. Setting it to 0 completely disables the caching system,
@@ -102,15 +105,15 b' All options with a [no] prepended can be specified in negated form'
 	because otherwise you'll spend more time re-flushing a too small cache
 	than working.
 
-    -classic, cl 
+    --classic
 	Gives IPython a similar feel to the classic Python
 	prompt.
 
-    -colors <scheme> 
+    colors=<scheme> 
 	Color scheme for prompts and exception reporting. Currently
 	implemented: NoColor, Linux and LightBG.
 
-    -[no]color_info 
+    --[no-]color_info 
 	IPython can display information about objects via a set of functions,
 	and optionally can use colors for this, syntax highlighting source
 	code and various other elements.  However, because this information is
@@ -124,12 +127,12 b' All options with a [no] prepended can be specified in negated form'
 	system. The magic function %color_info allows you to toggle this
 	interactively for testing.
 
-    -[no]debug 
+    --[no-]debug 
 	Show information about the loading process. Very useful to pin down
 	problems with your configuration files or to get details about
 	session restores.
 
-    -[no]deep_reload: 
+    --[no-]deep_reload: 
 	IPython can use the deep_reload module which reloads changes in
 	modules recursively (it replaces the reload() function, so you don't
 	need to change anything to use it).  deep_reload() forces a full
@@ -141,7 +144,7 b' All options with a [no] prepended can be specified in negated form'
 	feature is off by default [which means that you have both
 	normal reload() and dreload()].
 
-    -editor <name>
+    editor=<name>
 	Which editor to use with the %edit command. By default,
 	IPython will honor your EDITOR environment variable (if not
 	set, vi is the Unix default and notepad the Windows one).
@@ -150,7 +153,7 b' All options with a [no] prepended can be specified in negated form'
 	small, lightweight editor here (in case your default EDITOR is
 	something like Emacs).
 
-    -ipythondir <name> 
+    ipython_dir=<name> 
 	name of your IPython configuration directory IPYTHON_DIR. This
 	can also be specified through the environment variable
 	IPYTHON_DIR.
@@ -187,28 +190,28 b' All options with a [no] prepended can be specified in negated form'
       our first attempts failed because of inherent limitations of
       Python's Pickle module, so this may have to wait.
 
-    -[no]messages   
+    --[no-]messages   
 	Print messages which IPython collects about its startup
 	process (default on).
 
-    -[no]pdb
+    --[no-]pdb
 	Automatically call the pdb debugger after every uncaught
 	exception. If you are used to debugging using pdb, this puts
 	you automatically inside of it after any call (either in
 	IPython or in code called by it) which triggers an exception
 	which goes uncaught.
 
-    -pydb
+    --pydb
 	Makes IPython use the third party "pydb" package as debugger,
 	instead of pdb. Requires that pydb is installed.
 	
-    -[no]pprint
+    --[no-]pprint
 	ipython can optionally use the pprint (pretty printer) module
 	for displaying results. pprint tends to give a nicer display
 	of nested data structures. If you like it, you can turn it on
 	permanently in your config file (default off).
 
-    -profile, p <name>
+    profile=<name>
 
 	assume that your config file is ipythonrc-<name> or
 	ipy_profile_<name>.py (looks in current dir first, then in
@@ -227,7 +230,7 b' All options with a [no] prepended can be specified in negated form'
 	circular file inclusions, IPython will stop if it reaches 15
 	recursive inclusions.
 
-    -prompt_in1, pi1 <string>
+    pi1=<string>
 
         Specify the string used for input prompts. Note that if you are using
 	numbered prompts, the number is represented with a '\#' in the
@@ -236,7 +239,7 b' All options with a [no] prepended can be specified in negated form'
 	discusses in detail all the available escapes to customize your
 	prompts.
 
-    -prompt_in2, pi2 <string>
+    pi2=<string>
 	Similar to the previous option, but used for the continuation
 	prompts. The special sequence '\D' is similar to '\#', but
 	with all digits replaced dots (so you can have your
@@ -244,21 +247,22 b' All options with a [no] prepended can be specified in negated form'
 	' .\D.:' (note three spaces at the start for alignment with
 	'In [\#]').
 
-    -prompt_out,po <string> 
+    po=<string> 
 	String used for output prompts, also uses numbers like
 	prompt_in1. Default: 'Out[\#]:'
 
-    -quick   start in bare bones mode (no config file loaded).
+    --quick
+    start in bare bones mode (no config file loaded).
 
-    -rcfile <name>   
+    config_file=<name>   
 	name of your IPython resource configuration file. Normally
-	IPython loads ipythonrc (from current directory) or
-	IPYTHON_DIR/ipythonrc.
+	IPython loads ipython_config.py (from current directory) or
+	IPYTHON_DIR/profile_default.
 
 	If the loading of your config file fails, IPython starts with
 	a bare bones configuration (no modules loaded at all).
 
-    -[no]readline
+    --[no-]readline
 	use the readline library, which is needed to support name
 	completion and command history, among other things.  It is
 	enabled by default, but may cause problems for users of
@@ -268,7 +272,7 b' All options with a [no] prepended can be specified in negated form'
 	IPython's readline and syntax coloring fine, only 'emacs' (M-x
 	shell and C-c !) buffers do not.
 
-    -screen_length, sl <n>
+    sl=<n>
 	number of lines of your screen. This is used to control
 	printing of very long strings. Strings longer than this number
 	of lines will be sent through a pager instead of directly
@@ -281,41 +285,38 b' All options with a [no] prepended can be specified in negated form'
 	reason this isn't working well (it needs curses support), specify
 	it yourself. Otherwise don't change the default.
 
-    -separate_in, si <string>
+    si=<string>
 
 	separator before input prompts.
 	Default: '\n'
 
-    -separate_out, so <string>  
+    so=<string>  
         separator before output prompts.
         Default: nothing.
 
-    -separate_out2, so2
+    so2=<string>
 	separator after output prompts.
 	Default: nothing.
 	For these three options, use the value 0 to specify no separator.
 
-    -nosep   
+    --nosep
 	shorthand for '-SeparateIn 0 -SeparateOut 0 -SeparateOut2
 	0'. Simply removes all input/output separators.
 
-    -upgrade 
-	allows you to upgrade your IPYTHON_DIR configuration when you
+    --init
+	allows you to initialize your IPYTHON_DIR configuration when you
 	install a new version of IPython. Since new versions may
 	include new command line options or example files, this copies
-	updated ipythonrc-type files. However, it backs up (with a
+	updated config files. However, it backs up (with a
 	.old extension) all files which it overwrites so that you can
 	merge back any customizations you might have in your personal
 	files. Note that you should probably use %upgrade instead,
 	it's a safer alternative.
 
 
-    -Version    print version information and exit.
-
-    -wxversion <string> 
-    Deprecated.
+    --version    print version information and exit.
 
-    -xmode <modename>
+    xmode=<modename>
 
 	Mode for exception reporting.
 

docs/source/interactive/tutorial.txt

@@ -324,7 +324,7 b' made a profile for my project (which is called "parkfield")::'
 
 I also added a shell alias for convenience::
 
-    alias parkfield="ipython -pylab -profile parkfield" 
+    alias parkfield="ipython --pylab profile=parkfield" 
 
 Now I have a nice little directory with everything I ever type in,
 organized by project and date.

docs/source/parallel/parallel_intro.txt

@@ -195,7 +195,7 b' simply start a controller and engines on a single host using the'
 :command:`ipcluster` command. To start a controller and 4 engines on your
 localhost, just do::
 
-    $ ipcluster start -n 4
+    $ ipcluster start n=4
 
 More details about starting the IPython controller and engines can be found
 :ref:`here <parallel_process>`

docs/source/parallel/parallel_mpi.txt

@@ -53,11 +53,11 b' these things to happen.'
 Automatic starting using :command:`mpiexec` and :command:`ipcluster`
 --------------------------------------------------------------------
 
-The easiest approach is to use the `mpiexec` mode of :command:`ipcluster`,
+The easiest approach is to use the `MPIExec` Launchers in :command:`ipcluster`,
 which will first start a controller and then a set of engines using
 :command:`mpiexec`::
 
-    $ ipcluster mpiexec -n 4
+    $ ipcluster start n=4 elauncher=MPIExecEngineSetLauncher
 
 This approach is best as interrupting :command:`ipcluster` will automatically
 stop and clean up the controller and engines.
@@ -68,14 +68,14 b' Manual starting using :command:`mpiexec`'
 If you want to start the IPython engines using the :command:`mpiexec`, just
 do::
 
-    $ mpiexec -n 4 ipengine --mpi=mpi4py
+    $ mpiexec n=4 ipengine mpi=mpi4py
 
 This requires that you already have a controller running and that the FURL
 files for the engines are in place. We also have built in support for
 PyTrilinos [PyTrilinos]_, which can be used (assuming is installed) by
 starting the engines with::
 
-    $ mpiexec -n 4 ipengine --mpi=pytrilinos
+    $ mpiexec n=4 ipengine mpi=pytrilinos
 
 Automatic starting using PBS and :command:`ipcluster`
 ------------------------------------------------------
@@ -110,7 +110,7 b' distributed array. Save the following text in a file called :file:`psum.py`:'
 
 Now, start an IPython cluster::
 
-    $ ipcluster start -p mpi -n 4
+    $ ipcluster start profile=mpi n=4
     
 .. note::
 

docs/source/parallel/parallel_multiengine.txt

@@ -19,7 +19,7 b' To follow along with this tutorial, you will need to start the IPython'
 controller and four IPython engines. The simplest way of doing this is to use
 the :command:`ipcluster` command::
 
-    $ ipcluster start -n 4
+    $ ipcluster start n=4
     
 For more detailed information about starting the controller and engines, see
 our :ref:`introduction <ip1par>` to using IPython for parallel computing.

docs/source/parallel/parallel_process.txt

@@ -57,7 +57,7 b' controller and engines in the following situations:'
 
 1. When the controller and engines are all run on localhost. This is useful
    for testing or running on a multicore computer.
-2. When engines are started using the :command:`mpirun` command that comes 
+2. When engines are started using the :command:`mpiexec` command that comes 
    with most MPI [MPI]_ implementations
 3. When engines are started using the PBS [PBS]_ batch system 
    (or other `qsub` systems, such as SGE).
@@ -68,7 +68,7 b' controller and engines in the following situations:'
 .. note::
 
     Currently :command:`ipcluster` requires that the
-    :file:`~/.ipython/cluster_<profile>/security` directory live on a shared filesystem that is
+    :file:`~/.ipython/profile_<name>/security` directory live on a shared filesystem that is
     seen by both the controller and engines. If you don't have a shared file
     system you will need to use :command:`ipcontroller` and
     :command:`ipengine` directly.
@@ -80,9 +80,9 b' The simplest way to use ipcluster requires no configuration, and will'
 launch a controller and a number of engines on the local machine. For instance, 
 to start one controller and 4 engines on localhost, just do::
 
-    $ ipcluster start -n 4
+    $ ipcluster start n=4
 
-To see other command line options for the local mode, do::
+To see other command line options, do::
 
     $ ipcluster -h
 
@@ -92,12 +92,12 b' Configuring an IPython cluster'
 
 Cluster configurations are stored as `profiles`.  You can create a new profile with::
 
-    $ ipcluster create -p myprofile
+    $ ipython profile create --parallel profile=myprofile
 
 This will create the directory :file:`IPYTHONDIR/cluster_myprofile`, and populate it
 with the default configuration files for the three IPython cluster commands. Once
 you edit those files, you can continue to call ipcluster/ipcontroller/ipengine
-with no arguments beyond ``-p myprofile``, and any configuration will be maintained.
+with no arguments beyond ``p=myprofile``, and any configuration will be maintained.
 
 There is no limit to the number of profiles you can have, so you can maintain a profile for each
 of your common use cases. The default profile will be used whenever the
@@ -112,7 +112,8 b' Using various batch systems with :command:`ipcluster`'
 
 :command:`ipcluster` has a notion of Launchers that can start controllers
 and engines with various remote execution schemes.  Currently supported
-models include `mpiexec`, PBS-style (Torque, SGE), and Windows HPC Server.
+models include :command:`ssh`, :command`mpiexec`, PBS-style (Torque, SGE),
+and Windows HPC Server.
 
 .. note::
 
@@ -132,7 +133,7 b' The mpiexec/mpirun mode is useful if you:'
 
 If these are satisfied, you can create a new profile::
 
-    $ ipcluster create -p mpi
+    $ ipython profile create --parallel profile=mpi
 
 and edit the file :file:`IPYTHONDIR/cluster_mpi/ipcluster_config.py`.
 
@@ -140,11 +141,11 b' There, instruct ipcluster to use the MPIExec launchers by adding the lines:'
 
 .. sourcecode:: python
 
-    c.Global.engine_launcher = 'IPython.parallel.apps.launcher.MPIExecEngineSetLauncher'
+    c.IPClusterEnginesApp.engine_launcher = 'IPython.parallel.apps.launcher.MPIExecEngineSetLauncher'
 
 If the default MPI configuration is correct, then you can now start your cluster, with::
 
-    $ ipcluster start -n 4 -p mpi
+    $ ipcluster start n=4 profile=mpi
 
 This does the following:
 
@@ -155,7 +156,7 b' If you have a reason to also start the Controller with mpi, you can specify:'
 
 .. sourcecode:: python
 
-    c.Global.controller_launcher = 'IPython.parallel.apps.launcher.MPIExecControllerLauncher'
+    c.IPClusterStartApp.controller_launcher = 'IPython.parallel.apps.launcher.MPIExecControllerLauncher'
 
 .. note::
 
@@ -189,7 +190,7 b' The PBS mode uses the Portable Batch System [PBS]_ to start the engines.'
 
 As usual, we will start by creating a fresh profile::
 
-    $ ipcluster create -p pbs
+    $ ipython profile create --parallel profile=pbs
 
 And in :file:`ipcluster_config.py`, we will select the PBS launchers for the controller
 and engines:
@@ -207,35 +208,32 b' to specify your own. Here is a sample PBS script template:'
     #PBS -N ipython
     #PBS -j oe
     #PBS -l walltime=00:10:00
-    #PBS -l nodes=${n/4}:ppn=4
-    #PBS -q $queue
+    #PBS -l nodes={n/4}:ppn=4
+    #PBS -q {queue}
 
-    cd $$PBS_O_WORKDIR
-    export PATH=$$HOME/usr/local/bin
-    export PYTHONPATH=$$HOME/usr/local/lib/python2.7/site-packages
-    /usr/local/bin/mpiexec -n ${n} ipengine --cluster_dir=${cluster_dir}
+    cd $PBS_O_WORKDIR
+    export PATH=$HOME/usr/local/bin
+    export PYTHONPATH=$HOME/usr/local/lib/python2.7/site-packages
+    /usr/local/bin/mpiexec -n {n} ipengine profile_dir={profile_dir}
 
 There are a few important points about this template:
 
-1. This template will be rendered at runtime using IPython's :mod:`Itpl` 
-   template engine.
+1. This template will be rendered at runtime using IPython's :class:`EvalFormatter`.
+   This is simply a subclass of :class:`string.Formatter` that allows simple expressions
+   on keys.
 
 2. Instead of putting in the actual number of engines, use the notation
-   ``${n}`` to indicate the number of engines to be started. You can also uses
-   expressions like ``${n/4}`` in the template to indicate the number of 
-   nodes. There will always be a ${n} and ${cluster_dir} variable passed to the template.
+   ``{n}`` to indicate the number of engines to be started. You can also use
+   expressions like ``{n/4}`` in the template to indicate the number of nodes.
+   There will always be ``{n}`` and ``{profile_dir}`` variables passed to the formatter.
    These allow the batch system to know how many engines, and where the configuration
-   files reside. The same is true for the batch queue, with the template variable ``$queue``.
+   files reside. The same is true for the batch queue, with the template variable
+   ``{queue}``.
 
-3. Because ``$`` is a special character used by the template engine, you must  
-   escape any ``$`` by using ``$$``.  This is important when referring to 
-   environment variables in the template, or in SGE, where the config lines start
-   with ``#$``, which will have to be ``#$$``.
-
-4. Any options to :command:`ipengine` can be given in the batch script 
+3. Any options to :command:`ipengine` can be given in the batch script
    template, or in :file:`ipengine_config.py`.
 
-5. Depending on the configuration of you system, you may have to set 
+4. Depending on the configuration of you system, you may have to set
    environment variables in the script template.
 
 The controller template should be similar, but simpler:
@@ -246,12 +244,12 b' The controller template should be similar, but simpler:'
     #PBS -j oe
     #PBS -l walltime=00:10:00
     #PBS -l nodes=1:ppn=4
-    #PBS -q $queue
+    #PBS -q {queue}
 
-    cd $$PBS_O_WORKDIR
-    export PATH=$$HOME/usr/local/bin
-    export PYTHONPATH=$$HOME/usr/local/lib/python2.7/site-packages
-    ipcontroller --cluster_dir=${cluster_dir}
+    cd $PBS_O_WORKDIR
+    export PATH=$HOME/usr/local/bin
+    export PYTHONPATH=$HOME/usr/local/lib/python2.7/site-packages
+    ipcontroller profile_dir={profile_dir}
 
 
 Once you have created these scripts, save them with names like
@@ -267,14 +265,14 b' Once you have created these scripts, save them with names like'
 Alternately, you can just define the templates as strings inside :file:`ipcluster_config`.
 
 Whether you are using your own templates or our defaults, the extra configurables available are
-the number of engines to launch (``$n``, and the batch system queue to which the jobs are to be
-submitted (``$queue``)). These are configurables, and can be specified in
+the number of engines to launch (``{n}``, and the batch system queue to which the jobs are to be
+submitted (``{queue}``)). These are configurables, and can be specified in
 :file:`ipcluster_config`:
 
 .. sourcecode:: python
 
     c.PBSLauncher.queue = 'veryshort.q'
-    c.PBSEngineSetLauncher.n = 64
+    c.IPClusterEnginesApp.n = 64
 
 Note that assuming you are running PBS on a multi-node cluster, the Controller's default behavior
 of listening only on localhost is likely too restrictive.  In this case, also assuming the
@@ -287,7 +285,7 b' connections on all its interfaces, by adding in :file:`ipcontroller_config`:'
 
 You can now run the cluster with::
 
-    $ ipcluster start -p pbs -n 128
+    $ ipcluster start profile=pbs n=128
 
 Additional configuration options can be found in the PBS section of :file:`ipcluster_config`.
 
@@ -312,7 +310,7 b' nodes and :command:`ipcontroller` can be run remotely as well, or on localhost.'
 
 As usual, we start by creating a clean profile::
 
-    $ ipcluster create -p ssh
+    $ ipython profile create --parallel profile=ssh
 
 To use this mode, select the SSH launchers in :file:`ipcluster_config.py`:
 
@@ -334,8 +332,8 b" The controller's remote location and configuration can be specified:"
     # Set the arguments to be passed to ipcontroller
     # note that remotely launched ipcontroller will not get the contents of 
     # the local ipcontroller_config.py unless it resides on the *remote host*
-    # in the location specified by the --cluster_dir argument.
-    # c.SSHControllerLauncher.program_args = ['-r', '-ip', '0.0.0.0', '--cluster_dir', '/path/to/cd']
+    # in the location specified by the `profile_dir` argument.
+    # c.SSHControllerLauncher.program_args = ['--reuse', 'ip=0.0.0.0', 'profile_dir=/path/to/cd']
 
 .. note::
 
@@ -351,7 +349,7 b' on that host.'
 
     c.SSHEngineSetLauncher.engines = { 'host1.example.com' : 2,
                 'host2.example.com' : 5,
-                'host3.example.com' : (1, ['--cluster_dir', '/home/different/location']),
+                'host3.example.com' : (1, ['profile_dir=/home/different/location']),
                 'host4.example.com' : 8 }
 
 * The `engines` dict, where the keys are the host we want to run engines on and
@@ -364,7 +362,7 b' a single location:'
 
 .. sourcecode:: python
 
-    c.SSHEngineSetLauncher.engine_args = ['--cluster_dir', '/path/to/cluster_ssh']
+    c.SSHEngineSetLauncher.engine_args = ['profile_dir=/path/to/cluster_ssh']
 
 Current limitations of the SSH mode of :command:`ipcluster` are:
 
@@ -419,7 +417,7 b' When the controller and engines are running on different hosts, things are'
 slightly more complicated, but the underlying ideas are the same:
 
 1. Start the controller on a host using :command:`ipcontroller`.
-2. Copy :file:`ipcontroller-engine.json` from :file:`~/.ipython/cluster_<profile>/security` on
+2. Copy :file:`ipcontroller-engine.json` from :file:`~/.ipython/profile_<name>/security` on
    the controller's host to the host where the engines will run.
 3. Use :command:`ipengine` on the engine's hosts to start the engines.
 
@@ -427,7 +425,7 b' The only thing you have to be careful of is to tell :command:`ipengine` where'
 the :file:`ipcontroller-engine.json` file is located. There are two ways you
 can do this:
 
-* Put :file:`ipcontroller-engine.json` in the :file:`~/.ipython/cluster_<profile>/security`
+* Put :file:`ipcontroller-engine.json` in the :file:`~/.ipython/profile_<name>/security`
   directory on the engine's host, where it will be found automatically.
 * Call :command:`ipengine` with the ``--file=full_path_to_the_file``
   flag.
@@ -439,7 +437,7 b' The ``--file`` flag works like this::'
 .. note::
     
     If the controller's and engine's hosts all have a shared file system  
-    (:file:`~/.ipython/cluster_<profile>/security` is the same on all of them), then things
+    (:file:`~/.ipython/profile_<name>/security` is the same on all of them), then things
     will just work!
 
 Make JSON files persistent
@@ -452,10 +450,10 b' you want to unlock the door and enter your house. As with your house, you want'
 to be able to create the key (or JSON file) once, and then simply use it at
 any point in the future.
 
-To do this, the only thing you have to do is specify the `-r` flag, so that
+To do this, the only thing you have to do is specify the `--reuse` flag, so that
 the connection information in the JSON files remains accurate::
 
-    $ ipcontroller -r
+    $ ipcontroller --reuse
 
 Then, just copy the JSON files over the first time and you are set. You can
 start and stop the controller and engines any many times as you want in the
@@ -474,7 +472,7 b' Log files'
 
 All of the components of IPython have log files associated with them.
 These log files can be extremely useful in debugging problems with
-IPython and can be found in the directory :file:`~/.ipython/cluster_<profile>/log`.
+IPython and can be found in the directory :file:`~/.ipython/profile_<name>/log`.
 Sending the log files to us will often help us to debug any problems.
 
 

docs/source/parallel/parallel_security.txt

@@ -15,8 +15,8 b" computing. This feature brings up the important question of IPython's security"
 model. This document gives details about this model and how it is implemented
 in IPython's architecture.
 
-Processs and network topology
-=============================
+Process and network topology
+============================
 
 To enable parallel computing, IPython has a number of different processes that
 run. These processes are discussed at length in the IPython documentation and
@@ -36,15 +36,9 b' are summarized here:'
   interactive Python process that is used to coordinate the
   engines to get a parallel computation done.
 
-Collectively, these processes are called the IPython *kernel*, and the hub and schedulers
+Collectively, these processes are called the IPython *cluster*, and the hub and schedulers
 together are referred to as the *controller*.
 
-.. note::
-
-    Are these really still referred to as the Kernel?  It doesn't seem so to me.  'cluster'
-    seems more accurate.
-    
-    -MinRK
 
 These processes communicate over any transport supported by ZeroMQ (tcp,pgm,infiniband,ipc)
 with a well defined topology. The IPython hub and schedulers listen on sockets. Upon
@@ -118,20 +112,23 b' controller were on loopback on the connecting machine.'
 Authentication
 --------------
 
-To protect users of shared machines, an execution key is used to authenticate all messages.
+To protect users of shared machines, [HMAC]_ digests are used to sign messages, using a
+shared key.
 
 The Session object that handles the message protocol uses a unique key to verify valid
 messages. This can be any value specified by the user, but the default behavior is a
-pseudo-random 128-bit number, as generated by `uuid.uuid4()`. This key is checked on every
-message everywhere it is unpacked (Controller, Engine, and Client) to ensure that it came
-from an authentic user, and no messages that do not contain this key are acted upon in any
-way.
-
-There is exactly one key per cluster - it must be the same everywhere. Typically, the
-controller creates this key, and stores it in the private connection files
+pseudo-random 128-bit number, as generated by `uuid.uuid4()`. This key is used to
+initialize an HMAC object, which digests all messages, and includes that digest as a
+signature and part of the message. Every message that is unpacked (on Controller, Engine,
+and Client) will also be digested by the receiver, ensuring that the sender's key is the
+same as the receiver's. No messages that do not contain this key are acted upon in any
+way. The key itself is never sent over the network.
+
+There is exactly one shared key per cluster - it must be the same everywhere. Typically,
+the controller creates this key, and stores it in the private connection files
 `ipython-{engine|client}.json`. These files are typically stored in the
-`~/.ipython/cluster_<profile>/security` directory, and are maintained as readable only by
-the owner, just as is common practice with a user's keys in their `.ssh` directory.
+`~/.ipython/profile_<name>/security` directory, and are maintained as readable only by the
+owner, just as is common practice with a user's keys in their `.ssh` directory.
 
 .. warning::
 
@@ -171,13 +168,15 b' It is highly unlikely that an execution key could be guessed by an attacker'
 in a brute force guessing attack. A given instance of the IPython controller
 only runs for a relatively short amount of time (on the order of hours). Thus
 an attacker would have only a limited amount of time to test a search space of
-size 2**128.  
+size 2**128.  For added security, users can have arbitrarily long keys.
 
 .. warning::
 
-    If the attacker has gained enough access to intercept loopback connections on
-    *either* the controller or client, then the key is easily deduced from network
-    traffic.
+    If the attacker has gained enough access to intercept loopback connections on *either* the
+    controller or client, then a duplicate message can be sent. To protect against this,
+    recipients only allow each signature once, and consider duplicates invalid.  However,
+    the duplicate message could be sent to *another* recipient using the same key,
+    and it would be considered valid.
 
 
 Unauthorized engines
@@ -322,3 +321,4 b' channel is established.'
 
 .. [OpenSSH] <http://www.openssh.com/>
 .. [Paramiko] <http://www.lag.net/paramiko/>
+.. [HMAC] <http://tools.ietf.org/html/rfc2104.html>

docs/source/parallel/parallel_task.txt

@@ -24,7 +24,7 b' To follow along with this tutorial, you will need to start the IPython'
 controller and four IPython engines. The simplest way of doing this is to use
 the :command:`ipcluster` command::
 
-	$ ipcluster start -n 4
+	$ ipcluster start n=4
 	
 For more detailed information about starting the controller and engines, see
 our :ref:`introduction <ip1par>` to using IPython for parallel computing.
@@ -342,17 +342,17 b' Schedulers'
 
 There are a variety of valid ways to determine where jobs should be assigned in a 
 load-balancing situation.  In IPython, we support several standard schemes, and
-even make it easy to define your own.  The scheme can be selected via the ``--scheme``
-argument to :command:`ipcontroller`, or in the :attr:`HubFactory.scheme` attribute
+even make it easy to define your own.  The scheme can be selected via the ``scheme``
+argument to :command:`ipcontroller`, or in the :attr:`TaskScheduler.schemename` attribute
 of a controller config object.
 
 The built-in routing schemes:
 
 To select one of these schemes, simply do::
 
-    $ ipcontroller --scheme <schemename>
+    $ ipcontroller scheme=<schemename>
     for instance:
-    $ ipcontroller --scheme lru
+    $ ipcontroller scheme=lru
 
 lru: Least Recently Used
 

docs/source/parallel/parallel_winhpc.txt

@@ -162,7 +162,7 b' cluster using the Windows HPC Server 2008 job scheduler. To make sure that'
 to start an IPython cluster on your local host. To do this, open a Windows
 Command Prompt and type the following command::
 
-    ipcluster start -n 2
+    ipcluster start n=2
 
 You should see a number of messages printed to the screen, ending with 
 "IPython cluster: started". The result should look something like the following
@@ -179,11 +179,11 b' describe how to configure and run an IPython cluster on an actual compute'
 cluster running Windows HPC Server 2008. Here is an outline of the needed
 steps:
 
-1. Create a cluster profile using: ``ipcluster create -p mycluster``
+1. Create a cluster profile using: ``ipython profile create --parallel profile=mycluster``
 
 2. Edit configuration files in the directory :file:`.ipython\\cluster_mycluster`
 
-3. Start the cluster using: ``ipcluser start -p mycluster -n 32``
+3. Start the cluster using: ``ipcluser start profile=mycluster n=32``
 
 Creating a cluster profile
 --------------------------
@@ -198,13 +198,13 b' directory is a specially named directory (typically located in the'
 :file:`.ipython` subdirectory of your home directory) that contains the
 configuration files for a particular cluster profile, as well as log files and
 security keys. The naming convention for cluster directories is:
-:file:`cluster_<profile name>`. Thus, the cluster directory for a profile named
+:file:`profile_<profile name>`. Thus, the cluster directory for a profile named
 "foo" would be :file:`.ipython\\cluster_foo`.
 
 To create a new cluster profile (named "mycluster") and the associated cluster
 directory, type the following command at the Windows Command Prompt::
 
-    ipcluster create -p mycluster
+    ipython profile create --parallel profile=mycluster
 
 The output of this command is shown in the screenshot below. Notice how
 :command:`ipcluster` prints out the location of the newly created cluster
@@ -257,7 +257,7 b' Starting the cluster profile'
 Once a cluster profile has been configured, starting an IPython cluster using
 the profile is simple::
 
-    ipcluster start -p mycluster -n 32
+    ipcluster start profile=mycluster n=32
 
 The ``-n`` option tells :command:`ipcluster` how many engines to start (in
 this case 32). Stopping the cluster is as simple as typing Control-C.

setup.py

@@ -213,7 +213,7 b" if 'setuptools' in sys.modules:"
     setuptools_extra_args['entry_points'] = find_scripts(True)
     setup_args['extras_require'] = dict(
         parallel = 'pyzmq>=2.1.4',
-        zmq = 'pyzmq>=2.0.10.1',
+        zmq = 'pyzmq>=2.1.4',
         doc='Sphinx>=0.3',
         test='nose>=0.10.1',
     )

setupbase.py

@@ -103,7 +103,7 b' def find_packages():'
     Find all of IPython's packages.
     """
     packages = ['IPython']
-    add_package(packages, 'config', tests=True, others=['default','profile'])
+    add_package(packages, 'config', tests=True, others=['profile'])
     add_package(packages, 'core', tests=True)
     add_package(packages, 'deathrow', tests=True)
     add_package(packages, 'extensions')
@@ -150,8 +150,8 b' def find_package_data():'
     # This is not enough for these things to appear in an sdist.
     # We need to muck with the MANIFEST to get this to work
     package_data = {
-        'IPython.config.userconfig' : ['*'],
-        'IPython.testing' : ['*.txt']
+        'IPython.config.profile' : ['README', '*/*.py'],
+        'IPython.testing' : ['*.txt'],
     }
     return package_data
 
@@ -280,7 +280,7 b' def find_scripts(entry_points=False):'
             'irunner = IPython.lib.irunner:main'
         ]
         gui_scripts = [
-            'ipython-qtconsole = IPython.frontend.qt.console.ipythonqt:main',
+            'ipython-qtconsole = IPython.frontend.qt.console.qtconsoleapp:main',
         ]
         scripts = dict(console_scripts=console_scripts, gui_scripts=gui_scripts)
     else:
@@ -292,7 +292,6 b' def find_scripts(entry_points=False):'
                    pjoin(parallel_scripts, 'ipcluster'),
                    pjoin(parallel_scripts, 'iplogger'),
                    pjoin(main_scripts, 'ipython'),
-                   pjoin(main_scripts, 'ipython-qtconsole'),
                    pjoin(main_scripts, 'pycolor'),
                    pjoin(main_scripts, 'irunner'),
                    pjoin(main_scripts, 'iptest')

setupext/setupext.py

@@ -139,8 +139,9 b' def check_for_pyzmq():'
         print_status('pyzmq', "no (required for qtconsole and parallel computing capabilities)")
         return False
     else:
-        if zmq.__version__ < '2.0.10':
-            print_status('pyzmq', "no (require >= 2.0.10 for qtconsole and parallel computing capabilities)")
+        if zmq.__version__ < '2.1.4':
+            print_status('pyzmq', "no (have %s, but require >= 2.1.4 for"
+            " qtconsole and parallel computing capabilities)"%zmq.__version__)
             
         else:
             print_status("pyzmq", zmq.__version__)

IPython/config/default/ipcluster_config.py

@@ -1,241 +0,0 b''
-import os
-
-c = get_config()
-
-#-----------------------------------------------------------------------------
-# Select which launchers to use
-#-----------------------------------------------------------------------------
-
-# This allows you to control what method is used to start the controller
-# and engines.  The following methods are currently supported:
-# - Start as a regular process on localhost.
-# - Start using mpiexec.
-# - Start using the Windows HPC Server 2008 scheduler
-# - Start using PBS/SGE
-# - Start using SSH
-
-
-# The selected launchers can be configured below.
-
-# Options are:
-# - LocalControllerLauncher
-# - MPIExecControllerLauncher
-# - PBSControllerLauncher
-# - SGEControllerLauncher
-# - WindowsHPCControllerLauncher
-# c.Global.controller_launcher = 'IPython.parallel.apps.launcher.LocalControllerLauncher'
-# c.Global.controller_launcher = 'IPython.parallel.apps.launcher.PBSControllerLauncher'
-
-# Options are:
-# - LocalEngineSetLauncher
-# - MPIExecEngineSetLauncher
-# - PBSEngineSetLauncher
-# - SGEEngineSetLauncher
-# - WindowsHPCEngineSetLauncher
-# c.Global.engine_launcher = 'IPython.parallel.apps.launcher.LocalEngineSetLauncher'
-
-#-----------------------------------------------------------------------------
-# Global configuration
-#-----------------------------------------------------------------------------
-
-# The default number of engines that will be started. This is overridden by
-# the -n command line option: "ipcluster start -n 4"
-# c.Global.n = 2
-
-# Log to a file in cluster_dir/log, otherwise just log to sys.stdout.
-# c.Global.log_to_file = False
-
-# Remove old logs from cluster_dir/log before starting.
-# c.Global.clean_logs = True
-
-# The working directory for the process. The application will use os.chdir
-# to change to this directory before starting.
-# c.Global.work_dir = os.getcwd()
-
-
-#-----------------------------------------------------------------------------
-# Local process launchers
-#-----------------------------------------------------------------------------
-
-# The command line arguments to call the controller with.
-# c.LocalControllerLauncher.controller_args = \
-#    ['--log-to-file','--log-level', '40']
-
-# The working directory for the controller
-# c.LocalEngineSetLauncher.work_dir = u''
-
-# Command line argument passed to the engines.
-# c.LocalEngineSetLauncher.engine_args = ['--log-to-file','--log-level', '40']
-
-#-----------------------------------------------------------------------------
-# MPIExec launchers
-#-----------------------------------------------------------------------------
-
-# The mpiexec/mpirun command to use in both the controller and engines.
-# c.MPIExecLauncher.mpi_cmd = ['mpiexec']
-
-# Additional arguments to pass to the actual mpiexec command.
-# c.MPIExecLauncher.mpi_args = []
-
-# The mpiexec/mpirun command and args can be overridden if they should be different
-# for controller and engines.
-# c.MPIExecControllerLauncher.mpi_cmd = ['mpiexec']
-# c.MPIExecControllerLauncher.mpi_args = []
-# c.MPIExecEngineSetLauncher.mpi_cmd = ['mpiexec']
-# c.MPIExecEngineSetLauncher.mpi_args = []
-
-# The command line argument to call the controller with.
-# c.MPIExecControllerLauncher.controller_args = \
-#     ['--log-to-file','--log-level', '40']
-
-# Command line argument passed to the engines.
-# c.MPIExecEngineSetLauncher.engine_args = ['--log-to-file','--log-level', '40']
-
-# The default number of engines to start if not given elsewhere.
-# c.MPIExecEngineSetLauncher.n = 1
-
-#-----------------------------------------------------------------------------
-# SSH launchers
-#-----------------------------------------------------------------------------
-
-# ipclusterz can be used to launch controller and engines remotely via ssh.
-# Note that currently ipclusterz does not do any file distribution, so if
-# machines are not on a shared filesystem, config and json files must be
-# distributed.  For this reason, the reuse_files defaults to True on an
-# ssh-launched Controller.  This flag can be overridded by the program_args
-# attribute of c.SSHControllerLauncher.
-
-# set the ssh cmd for launching remote commands. The default is ['ssh']
-# c.SSHLauncher.ssh_cmd = ['ssh']
-
-# set the ssh cmd for launching remote commands. The default is ['ssh']
-# c.SSHLauncher.ssh_args = ['tt']
-
-# Set the user and hostname for the controller
-# c.SSHControllerLauncher.hostname = 'controller.example.com'
-# c.SSHControllerLauncher.user = os.environ.get('USER','username')
-
-# Set the arguments to be passed to ipcontrollerz
-# note that remotely launched ipcontrollerz will not get the contents of 
-# the local ipcontrollerz_config.py unless it resides on the *remote host*
-# in the location specified by the --cluster_dir argument.
-# c.SSHControllerLauncher.program_args = ['-r', '-ip', '0.0.0.0', '--cluster_dir', '/path/to/cd']
-
-# Set the default args passed to ipenginez for SSH launched engines
-# c.SSHEngineSetLauncher.engine_args = ['--mpi', 'mpi4py']
-
-# SSH engines are launched as a dict of locations/n-engines.
-# if a value is a tuple instead of an int, it is assumed to be of the form
-# (n, [args]), setting the arguments to passed to ipenginez on `host`.
-# otherwise, c.SSHEngineSetLauncher.engine_args will be used as the default.
-
-# In this case, there will be 3 engines at my.example.com, and
-# 2 at [e-mail] with a special json connector location.
-# c.SSHEngineSetLauncher.engines = {'my.example.com' : 3,
-#                                   '[e-mail]' : (2, ['-f', '/path/to/ipcontroller-engine.json']}
-#                                  }
-
-#-----------------------------------------------------------------------------
-# Unix batch (PBS) schedulers launchers
-#-----------------------------------------------------------------------------
-
-# SGE and PBS are very similar. All configurables in this section called 'PBS*'
-# also exist as 'SGE*'.
-
-# The command line program to use to submit a PBS job.
-# c.PBSLauncher.submit_command = ['qsub']
-
-# The command line program to use to delete a PBS job.
-# c.PBSLauncher.delete_command = ['qdel']
-
-# The PBS queue in which the job should run
-# c.PBSLauncher.queue = 'myqueue'
-
-# A regular expression that takes the output of qsub and find the job id.
-# c.PBSLauncher.job_id_regexp = r'\d+'
-
-# If for some reason the Controller and Engines have different options above, they
-# can be set as c.PBSControllerLauncher.<option> etc.
-
-# PBS and SGE have default templates, but you can specify your own, either as strings
-# or from files, as described here:
-
-# The batch submission script used to start the controller. This is where
-# environment variables would be setup, etc. This string is interpreted using
-# the Itpl module in IPython.external. Basically, you can use ${n} for the
-# number of engine and ${cluster_dir} for the cluster_dir.
-# c.PBSControllerLauncher.batch_template = """
-# #PBS -N ipcontroller
-# #PBS -q $queue
-# 
-# ipcontrollerz --cluster-dir $cluster_dir
-# """
-
-# You can also load this template from a file
-# c.PBSControllerLauncher.batch_template_file = u"/path/to/my/template.sh"
-
-# The name of the instantiated batch script that will actually be used to
-# submit the job. This will be written to the cluster directory.
-# c.PBSControllerLauncher.batch_file_name = u'pbs_controller'
-
-# The batch submission script used to start the engines. This is where
-# environment variables would be setup, etc. This string is interpreted using
-# the Itpl module in IPython.external. Basically, you can use ${n} for the
-# number of engine and ${cluster_dir} for the cluster_dir.
-# c.PBSEngineSetLauncher.batch_template = """
-# #PBS -N ipcontroller
-# #PBS -l nprocs=$n
-# 
-# ipenginez --cluster-dir $cluster_dir$s
-# """
-
-# You can also load this template from a file
-# c.PBSControllerLauncher.batch_template_file = u"/path/to/my/template.sh"
-
-# The name of the instantiated batch script that will actually be used to
-# submit the job. This will be written to the cluster directory.
-# c.PBSEngineSetLauncher.batch_file_name = u'pbs_engines'
-
-
-
-#-----------------------------------------------------------------------------
-# Windows HPC Server 2008 launcher configuration
-#-----------------------------------------------------------------------------
-
-# c.IPControllerJob.job_name = 'IPController'
-# c.IPControllerJob.is_exclusive = False
-# c.IPControllerJob.username = r'USERDOMAIN\USERNAME'
-# c.IPControllerJob.priority = 'Highest'
-# c.IPControllerJob.requested_nodes = ''
-# c.IPControllerJob.project = 'MyProject'
-
-# c.IPControllerTask.task_name = 'IPController'
-# c.IPControllerTask.controller_cmd = [u'ipcontroller.exe']
-# c.IPControllerTask.controller_args = ['--log-to-file', '--log-level', '40']
-# c.IPControllerTask.environment_variables = {}
-
-# c.WindowsHPCControllerLauncher.scheduler = 'HEADNODE'
-# c.WindowsHPCControllerLauncher.job_file_name = u'ipcontroller_job.xml'
-
-
-# c.IPEngineSetJob.job_name = 'IPEngineSet'
-# c.IPEngineSetJob.is_exclusive = False
-# c.IPEngineSetJob.username = r'USERDOMAIN\USERNAME'
-# c.IPEngineSetJob.priority = 'Highest'
-# c.IPEngineSetJob.requested_nodes = ''
-# c.IPEngineSetJob.project = 'MyProject'
-
-# c.IPEngineTask.task_name = 'IPEngine'
-# c.IPEngineTask.engine_cmd = [u'ipengine.exe']
-# c.IPEngineTask.engine_args = ['--log-to-file', '--log-level', '40']
-# c.IPEngineTask.environment_variables = {}
-
-# c.WindowsHPCEngineSetLauncher.scheduler = 'HEADNODE'
-# c.WindowsHPCEngineSetLauncher.job_file_name = u'ipengineset_job.xml'
-
-
-
-
-
-
-

IPython/config/default/ipcontroller_config.py

@@ -1,180 +0,0 b''
-from IPython.config.loader import Config
-
-c = get_config()
-
-#-----------------------------------------------------------------------------
-# Global configuration
-#-----------------------------------------------------------------------------
-
-# Basic Global config attributes
-
-# Start up messages are logged to stdout using the logging module.
-# These all happen before the twisted reactor is started and are
-# useful for debugging purposes. Can be (10=DEBUG,20=INFO,30=WARN,40=CRITICAL) 
-# and smaller is more verbose.
-# c.Global.log_level = 20
-
-# Log to a file in cluster_dir/log, otherwise just log to sys.stdout.
-# c.Global.log_to_file = False
-
-# Remove old logs from cluster_dir/log before starting.
-# c.Global.clean_logs = True
-
-# A list of Python statements that will be run before starting the 
-# controller. This is provided because occasionally certain things need to 
-# be imported in the controller for pickling to work.
-# c.Global.import_statements = ['import math']
-
-# Reuse the controller's JSON files. If False, JSON files are regenerated
-# each time the controller is run. If True, they will be reused, *but*, you
-# also must set the network ports by hand. If set, this will override the
-# values set for the client and engine connections below.
-# c.Global.reuse_files = True
-
-# Enable exec_key authentication on all messages. Default is True
-# c.Global.secure = True
-
-# The working directory for the process. The application will use os.chdir
-# to change to this directory before starting.
-# c.Global.work_dir = os.getcwd()
-
-# The log url for logging to an `iploggerz` application.  This will override
-# log-to-file.
-# c.Global.log_url = 'tcp://127.0.0.1:20202'
-
-# The specific external IP that is used to disambiguate multi-interface URLs.
-# The default behavior is to guess from external IPs gleaned from `socket`.
-# c.Global.location = '192.168.1.123'
-
-# The ssh server remote clients should use to connect to this controller.
-# It must be a machine that can see the interface specified in client_ip.
-# The default for client_ip is localhost, in which case the sshserver must
-# be an external IP of the controller machine.
-# c.Global.sshserver = 'controller.example.com'
-
-# the url to use for registration.  If set, this overrides engine-ip,
-# engine-transport client-ip,client-transport, and regport.
-# c.RegistrationFactory.url = 'tcp://*:12345'
-
-# the port to use for registration.  Clients and Engines both use this
-# port for registration.
-# c.RegistrationFactory.regport = 10101
-
-#-----------------------------------------------------------------------------
-# Configure the Task Scheduler
-#-----------------------------------------------------------------------------
-
-# The routing scheme. 'pure' will use the pure-ZMQ scheduler. Any other 
-# value will use a Python scheduler with various routing schemes.
-# python schemes are: lru, weighted, random, twobin. Default is 'weighted'.
-# Note that the pure ZMQ scheduler does not support many features, such as
-# dying engines, dependencies, or engine-subset load-balancing.
-# c.ControllerFactory.scheme = 'pure'
-
-# The Python scheduler can limit the number of outstanding tasks per engine
-# by using an HWM option.  This allows engines with long-running tasks
-# to not steal too many tasks from other engines. The default is 0, which
-# means agressively distribute messages, never waiting for them to finish.
-# c.TaskScheduler.hwm = 0
-
-# Whether to use Threads or Processes to start the Schedulers.  Threads will
-# use less resources, but potentially reduce throughput. Default is to 
-# use processes.  Note that the a Python scheduler will always be in a Process.
-# c.ControllerFactory.usethreads
-
-#-----------------------------------------------------------------------------
-# Configure the Hub
-#-----------------------------------------------------------------------------
-
-# Which class to use for the db backend.  Currently supported are DictDB (the
-# default), and MongoDB. Uncomment this line to enable MongoDB, which will
-# slow-down the Hub's responsiveness, but also reduce its memory footprint.
-# c.HubFactory.db_class = 'IPython.parallel.controller.mongodb.MongoDB'
-
-# The heartbeat ping frequency.  This is the frequency (in ms) at which the
-# Hub pings engines for heartbeats.  This determines how quickly the Hub
-# will react to engines coming and going.  A lower number means faster response
-# time, but more network activity.  The default is 100ms
-# c.HubFactory.ping = 100
-
-# HubFactory queue port pairs, to set by name: mux, iopub, control, task.  Set
-# each as a tuple of length 2 of ints.  The default is to find random
-# available ports
-# c.HubFactory.mux = (10102,10112)
-
-#-----------------------------------------------------------------------------
-# Configure the client connections
-#-----------------------------------------------------------------------------
-
-# Basic client connection config attributes
-
-# The network interface the controller will listen on for client connections.
-# This should be an IP address or interface on the controller. An asterisk
-# means listen on all interfaces. The transport can be any transport
-# supported by zeromq (tcp,epgm,pgm,ib,ipc):
-# c.HubFactory.client_ip = '*'
-# c.HubFactory.client_transport = 'tcp'
-
-# individual client ports to configure by name: query_port, notifier_port
-# c.HubFactory.query_port = 12345
-
-#-----------------------------------------------------------------------------
-# Configure the engine connections
-#-----------------------------------------------------------------------------
-
-# Basic config attributes for the engine connections.
-
-# The network interface the controller will listen on for engine connections.
-# This should be an IP address or interface on the controller. An asterisk
-# means listen on all interfaces. The transport can be any transport
-# supported by zeromq (tcp,epgm,pgm,ib,ipc):
-# c.HubFactory.engine_ip = '*'
-# c.HubFactory.engine_transport = 'tcp'
-
-# set the engine heartbeat ports to use:
-# c.HubFactory.hb = (10303,10313)
-
-#-----------------------------------------------------------------------------
-# Configure the TaskRecord database backend
-#-----------------------------------------------------------------------------
-
-# For memory/persistance reasons, tasks can be stored out-of-memory in a database.
-# Currently, only sqlite and mongodb are supported as backends, but the interface
-# is fairly simple, so advanced developers could write their own backend.
-
-# ----- in-memory configuration --------
-# this line restores the default behavior: in-memory storage of all results.
-# c.HubFactory.db_class = 'IPython.parallel.controller.dictdb.DictDB'
-
-# ----- sqlite configuration --------
-# use this line to activate sqlite:
-# c.HubFactory.db_class = 'IPython.parallel.controller.sqlitedb.SQLiteDB'
-
-# You can specify the name of the db-file.  By default, this will be located
-# in the active cluster_dir, e.g. ~/.ipython/clusterz_default/tasks.db
-# c.SQLiteDB.filename = 'tasks.db'
-
-# You can also specify the location of the db-file, if you want it to be somewhere
-# other than the cluster_dir.
-# c.SQLiteDB.location = '/scratch/'
-
-# This will specify the name of the table for the controller to use.  The default
-# behavior is to use the session ID of the SessionFactory object (a uuid). Overriding
-# this will result in results persisting for multiple sessions.
-# c.SQLiteDB.table = 'results'
-
-# ----- mongodb configuration --------
-# use this line to activate mongodb:
-# c.HubFactory.db_class = 'IPython.parallel.controller.mongodb.MongoDB'
-
-# You can specify the args and kwargs pymongo will use when creating the Connection.
-# For more information on what these options might be, see pymongo documentation.
-# c.MongoDB.connection_kwargs = {}
-# c.MongoDB.connection_args = []
-
-# This will specify the name of the mongo database for the controller to use.  The default
-# behavior is to use the session ID of the SessionFactory object (a uuid). Overriding
-# this will result in task results persisting through multiple sessions.
-# c.MongoDB.database = 'ipythondb'
-
-

IPython/config/default/ipengine_config.py

@@ -1,85 +0,0 b''
-c = get_config()
-
-#-----------------------------------------------------------------------------
-# Global configuration
-#-----------------------------------------------------------------------------
-
-# Start up messages are logged to stdout using the logging module.
-# These all happen before the twisted reactor is started and are
-# useful for debugging purposes. Can be (10=DEBUG,20=INFO,30=WARN,40=CRITICAL) 
-# and smaller is more verbose.
-# c.Global.log_level = 20
-
-# Log to a file in cluster_dir/log, otherwise just log to sys.stdout.
-# c.Global.log_to_file = False
-
-# Remove old logs from cluster_dir/log before starting.
-# c.Global.clean_logs = True
-
-# A list of strings that will be executed in the users namespace on the engine
-# before it connects to the controller.
-# c.Global.exec_lines = ['import numpy']
-
-# The engine will try to connect to the controller multiple times, to allow
-# the controller time to startup and write its FURL file. These parameters 
-# control the number of retries (connect_max_tries) and the initial delay
-# (connect_delay) between attemps. The actual delay between attempts gets
-# longer each time by a factor of 1.5 (delay[i] = 1.5*delay[i-1])
-# those attemps.
-# c.Global.connect_delay = 0.1
-# c.Global.connect_max_tries = 15
-
-# By default, the engine will look for the controller's JSON file in its own
-# cluster directory. Sometimes, the JSON file will be elsewhere and this 
-# attribute can be set to the full path of the JSON file.
-# c.Global.url_file = u'/path/to/my/ipcontroller-engine.json'
-
-# The working directory for the process. The application will use os.chdir
-# to change to this directory before starting.
-# c.Global.work_dir = os.getcwd()
-
-#-----------------------------------------------------------------------------
-# MPI configuration
-#-----------------------------------------------------------------------------
-
-# Upon starting the engine can be configured to call MPI_Init. This section
-# configures that.
-
-# Select which MPI section to execute to setup MPI. The value of this 
-# attribute must match the name of another attribute in the MPI config 
-# section (mpi4py, pytrilinos, etc.). This can also be set by the --mpi
-# command line option.
-# c.MPI.use = ''
-
-# Initialize MPI using mpi4py. To use this, set c.MPI.use = 'mpi4py' to use
-# --mpi=mpi4py at the command line.
-# c.MPI.mpi4py = """from mpi4py import MPI as mpi
-# mpi.size = mpi.COMM_WORLD.Get_size()
-# mpi.rank = mpi.COMM_WORLD.Get_rank()
-# """
-
-# Initialize MPI using pytrilinos. To use this, set c.MPI.use = 'pytrilinos' 
-# to use --mpi=pytrilinos at the command line.
-# c.MPI.pytrilinos = """from PyTrilinos import Epetra
-# class SimpleStruct:
-# pass
-# mpi = SimpleStruct()
-# mpi.rank = 0
-# mpi.size = 0
-# """
-
-#-----------------------------------------------------------------------------
-# Developer level configuration attributes
-#-----------------------------------------------------------------------------
-
-# You shouldn't have to modify anything in this section. These attributes
-# are more for developers who want to change the behavior of the controller
-# at a fundamental level.
-
-# You should not have to change these attributes.
-
-# c.Global.url_file_name = u'ipcontroller-engine.furl'
-
-
-
-

IPython/config/default/ipython_config.py

@@ -1,165 +0,0 b''
-# Get the config being loaded so we can set attributes on it
-c = get_config()
-
-#-----------------------------------------------------------------------------
-# Global options
-#-----------------------------------------------------------------------------
-
-# c.Global.display_banner = True
-
-# c.Global.classic = False
-
-# c.Global.nosep = True
-
-# If you still use multiple versions of IPytho on the same machine,
-# set this to True to suppress warnings about old configuration files
-# c.Global.ignore_old_config = False
-
-# Set this to determine the detail of what is logged at startup.
-# The default is 30 and possible values are 0,10,20,30,40,50.
-# c.Global.log_level = 20
-
-# This should be a list of importable Python modules that have an
-# load_ipython_extension(ip) method.  This method gets called when the extension
-# is loaded.  You can put your extensions anywhere they can be imported
-# but we add the extensions subdir of the ipython directory to sys.path
-# during extension loading, so you can put them there as well.
-# c.Global.extensions = [
-#     'myextension'
-# ]
-
-# These lines are run in IPython in the user's namespace after extensions 
-# are loaded.  They can contain full IPython syntax with magics etc.
-# c.Global.exec_lines = [
-#     'import numpy',
-#     'a = 10; b = 20',
-#     '1/0'
-# ]
-
-# These files are run in IPython in the user's namespace.  Files with a .py
-# extension need to be pure Python.  Files with a .ipy extension can have
-# custom IPython syntax (like magics, etc.).  
-# These files need to be in the cwd, the ipython_dir or be absolute paths.
-# c.Global.exec_files = [
-#     'mycode.py',
-#     'fancy.ipy'
-# ]
-
-#-----------------------------------------------------------------------------
-# InteractiveShell options
-#-----------------------------------------------------------------------------
-
-# c.InteractiveShell.autocall = 1
-
-# c.TerminalInteractiveShell.autoedit_syntax = False
-
-# c.InteractiveShell.autoindent = True
-
-# c.InteractiveShell.automagic = False
-
-# c.TerminalTerminalInteractiveShell.banner1 = 'This if for overriding the default IPython banner'
- 
-# c.TerminalTerminalInteractiveShell.banner2 = "This is for extra banner text"
-
-# c.InteractiveShell.cache_size = 1000
-
-# c.InteractiveShell.colors = 'LightBG'
-
-# c.InteractiveShell.color_info = True
-
-# c.TerminalInteractiveShell.confirm_exit = True
-
-# c.InteractiveShell.deep_reload = False
-
-# c.TerminalInteractiveShell.editor = 'nano'
-
-# c.InteractiveShell.logstart = True
-
-# c.InteractiveShell.logfile = u'ipython_log.py'
-
-# c.InteractiveShell.logappend = u'mylog.py'
-
-# c.InteractiveShell.object_info_string_level = 0
-
-# c.TerminalInteractiveShell.pager = 'less'
-
-# c.InteractiveShell.pdb = False
-
-# c.InteractiveShell.prompt_in1 = 'In [\#]: '
-# c.InteractiveShell.prompt_in2 = '   .\D.: '
-# c.InteractiveShell.prompt_out = 'Out[\#]: '
-# c.InteractiveShell.prompts_pad_left = True
-
-# c.InteractiveShell.quiet = False
-
-# c.InteractiveShell.history_length = 10000
-
-# Readline 
-# c.InteractiveShell.readline_use = True
-
-# be careful with meta-key ('\M-<x>') bindings, because
-# they conflict with 8-bit encodings (e.g. UTF8)
-
-# c.InteractiveShell.readline_parse_and_bind = [
-#     'tab: complete',
-#     '"\C-l": possible-completions',
-#     'set show-all-if-ambiguous on',
-#     '"\C-o": tab-insert',
-#     '"\C-r": reverse-search-history',
-#     '"\C-s": forward-search-history',
-#     '"\C-p": history-search-backward',
-#     '"\C-n": history-search-forward',
-#     '"\e[A": history-search-backward',
-#     '"\e[B": history-search-forward',
-#     '"\C-k": kill-line',
-#     '"\C-u": unix-line-discard',
-# ]
-# c.InteractiveShell.readline_remove_delims = '-/~'
-# c.InteractiveShell.readline_merge_completions = True
-# c.InteractiveShell.readline_omit__names = 0
-
-# c.TerminalInteractiveShell.screen_length = 0
-
-# c.InteractiveShell.separate_in = '\n'
-# c.InteractiveShell.separate_out = ''
-# c.InteractiveShell.separate_out2 = ''
-
-# c.TerminalInteractiveShell.term_title = False
-
-# c.InteractiveShell.wildcards_case_sensitive = True
-
-# c.InteractiveShell.xmode = 'Context'
-
-#-----------------------------------------------------------------------------
-# Formatter and display options
-#-----------------------------------------------------------------------------
-
-# c.PlainTextFormatter.pprint = True
-
-#-----------------------------------------------------------------------------
-# PrefilterManager options
-#-----------------------------------------------------------------------------
-
-# c.PrefilterManager.multi_line_specials = True
-
-#-----------------------------------------------------------------------------
-# AliasManager options
-#-----------------------------------------------------------------------------
-
-# Do this to disable all defaults
-# c.AliasManager.default_aliases = []
-
-# c.AliasManager.user_aliases = [
-#     ('foo', 'echo Hi')
-# ]
-
-#-----------------------------------------------------------------------------
-# HistoryManager options
-#-----------------------------------------------------------------------------
-
-# Enable logging output as well as input to the database.
-# c.HistoryManager.db_log_output = False
-
-# Only write to the database every n commands - this can save disk
-# access (and hence power) over the default of writing on every command.
-# c.HistoryManager.db_cache_size = 0

IPython/config/profile/ipython_config_numeric.py

@@ -1,20 +0,0 b''
-c = get_config()
-
-# This can be used at any point in a config file to load a sub config
-# and merge it into the current one.
-load_subconfig('ipython_config.py')
-
-lines = """
-import numpy
-import scipy
-import numpy as np
-import scipy as sp
-"""
-
-# You have to make sure that attributes that are containers already
-# exist before using them.  Simple assigning a new list will override
-# all previous values.
-if hasattr(c.Global, 'exec_lines'):
-    c.Global.exec_lines.append(lines)
-else:
-    c.Global.exec_lines = [lines]
  No newline at end of file

IPython/parallel/apps/clusterdir.py

@@ -1,566 +0,0 b''
-#!/usr/bin/env python
-# encoding: utf-8
-"""
-The IPython cluster directory
-"""
-
-#-----------------------------------------------------------------------------
-#  Copyright (C) 2008-2009  The IPython Development Team
-#
-#  Distributed under the terms of the BSD License.  The full license is in
-#  the file COPYING, distributed as part of this software.
-#-----------------------------------------------------------------------------
-
-#-----------------------------------------------------------------------------
-# Imports
-#-----------------------------------------------------------------------------
-
-from __future__ import with_statement
-
-import os
-import logging
-import re
-import shutil
-import sys
-
-from subprocess import Popen, PIPE
-
-from IPython.config.loader import PyFileConfigLoader
-from IPython.config.configurable import Configurable
-from IPython.core.application import Application, BaseAppConfigLoader
-from IPython.core.crashhandler import CrashHandler
-from IPython.core import release
-from IPython.utils.path import (
-    get_ipython_package_dir,
-    expand_path
-)
-from IPython.utils.traitlets import Unicode
-
-#-----------------------------------------------------------------------------
-# Module errors
-#-----------------------------------------------------------------------------
-
-class ClusterDirError(Exception):
-    pass
-
-
-class PIDFileError(Exception):
-    pass
-
-
-#-----------------------------------------------------------------------------
-# Class for managing cluster directories
-#-----------------------------------------------------------------------------
-
-class ClusterDir(Configurable):
-    """An object to manage the cluster directory and its resources.
-
-    The cluster directory is used by :command:`ipengine`, 
-    :command:`ipcontroller` and :command:`ipclsuter` to manage the
-    configuration, logging and security of these applications.
-
-    This object knows how to find, create and manage these directories. This
-    should be used by any code that want's to handle cluster directories.
-    """
-
-    security_dir_name = Unicode('security')
-    log_dir_name = Unicode('log')
-    pid_dir_name = Unicode('pid')
-    security_dir = Unicode(u'')
-    log_dir = Unicode(u'')
-    pid_dir = Unicode(u'')
-    location = Unicode(u'')
-
-    def __init__(self, location=u''):
-        super(ClusterDir, self).__init__(location=location)
-
-    def _location_changed(self, name, old, new):
-        if not os.path.isdir(new):
-            os.makedirs(new)
-        self.security_dir = os.path.join(new, self.security_dir_name)
-        self.log_dir = os.path.join(new, self.log_dir_name)
-        self.pid_dir = os.path.join(new, self.pid_dir_name)
-        self.check_dirs()
-
-    def _log_dir_changed(self, name, old, new):
-        self.check_log_dir()
-
-    def check_log_dir(self):
-        if not os.path.isdir(self.log_dir):
-            os.mkdir(self.log_dir)
-
-    def _security_dir_changed(self, name, old, new):
-        self.check_security_dir()
-
-    def check_security_dir(self):
-        if not os.path.isdir(self.security_dir):
-            os.mkdir(self.security_dir, 0700)
-        os.chmod(self.security_dir, 0700)
-
-    def _pid_dir_changed(self, name, old, new):
-        self.check_pid_dir()
-
-    def check_pid_dir(self):
-        if not os.path.isdir(self.pid_dir):
-            os.mkdir(self.pid_dir, 0700)
-        os.chmod(self.pid_dir, 0700)
-
-    def check_dirs(self):
-        self.check_security_dir()
-        self.check_log_dir()
-        self.check_pid_dir()
-
-    def load_config_file(self, filename):
-        """Load a config file from the top level of the cluster dir.
-
-        Parameters
-        ----------
-        filename : unicode or str
-            The filename only of the config file that must be located in
-            the top-level of the cluster directory.
-        """
-        loader = PyFileConfigLoader(filename, self.location)
-        return loader.load_config()
-
-    def copy_config_file(self, config_file, path=None, overwrite=False):
-        """Copy a default config file into the active cluster directory.
-
-        Default configuration files are kept in :mod:`IPython.config.default`.
-        This function moves these from that location to the working cluster
-        directory.
-        """
-        if path is None:
-            import IPython.config.default
-            path = IPython.config.default.__file__.split(os.path.sep)[:-1]
-            path = os.path.sep.join(path)
-        src = os.path.join(path, config_file)
-        dst = os.path.join(self.location, config_file)
-        if not os.path.isfile(dst) or overwrite:
-            shutil.copy(src, dst)
-
-    def copy_all_config_files(self, path=None, overwrite=False):
-        """Copy all config files into the active cluster directory."""
-        for f in [u'ipcontroller_config.py', u'ipengine_config.py',
-                  u'ipcluster_config.py']:
-            self.copy_config_file(f, path=path, overwrite=overwrite)
-
-    @classmethod
-    def create_cluster_dir(csl, cluster_dir):
-        """Create a new cluster directory given a full path.
-
-        Parameters
-        ----------
-        cluster_dir : str
-            The full path to the cluster directory.  If it does exist, it will
-            be used.  If not, it will be created.
-        """
-        return ClusterDir(location=cluster_dir)
-
-    @classmethod
-    def create_cluster_dir_by_profile(cls, path, profile=u'default'):
-        """Create a cluster dir by profile name and path.
-
-        Parameters
-        ----------
-        path : str
-            The path (directory) to put the cluster directory in.
-        profile : str
-            The name of the profile.  The name of the cluster directory will
-            be "cluster_<profile>".
-        """
-        if not os.path.isdir(path):
-            raise ClusterDirError('Directory not found: %s' % path)
-        cluster_dir = os.path.join(path, u'cluster_' + profile)
-        return ClusterDir(location=cluster_dir)
-
-    @classmethod
-    def find_cluster_dir_by_profile(cls, ipython_dir, profile=u'default'):
-        """Find an existing cluster dir by profile name, return its ClusterDir.
-
-        This searches through a sequence of paths for a cluster dir.  If it
-        is not found, a :class:`ClusterDirError` exception will be raised.
-
-        The search path algorithm is:
-        1. ``os.getcwd()``
-        2. ``ipython_dir``
-        3. The directories found in the ":" separated 
-           :env:`IPCLUSTER_DIR_PATH` environment variable.
-
-        Parameters
-        ----------
-        ipython_dir : unicode or str
-            The IPython directory to use.
-        profile : unicode or str
-            The name of the profile.  The name of the cluster directory
-            will be "cluster_<profile>".
-        """
-        dirname = u'cluster_' + profile
-        cluster_dir_paths = os.environ.get('IPCLUSTER_DIR_PATH','')
-        if cluster_dir_paths:
-            cluster_dir_paths = cluster_dir_paths.split(':')
-        else:
-            cluster_dir_paths = []
-        paths = [os.getcwd(), ipython_dir] + cluster_dir_paths
-        for p in paths:
-            cluster_dir = os.path.join(p, dirname)
-            if os.path.isdir(cluster_dir):
-                return ClusterDir(location=cluster_dir)
-        else:
-            raise ClusterDirError('Cluster directory not found in paths: %s' % dirname)
-
-    @classmethod
-    def find_cluster_dir(cls, cluster_dir):
-        """Find/create a cluster dir and return its ClusterDir.
-
-        This will create the cluster directory if it doesn't exist.
-
-        Parameters
-        ----------
-        cluster_dir : unicode or str
-            The path of the cluster directory.  This is expanded using
-            :func:`IPython.utils.genutils.expand_path`.
-        """
-        cluster_dir = expand_path(cluster_dir)
-        if not os.path.isdir(cluster_dir):
-            raise ClusterDirError('Cluster directory not found: %s' % cluster_dir)
-        return ClusterDir(location=cluster_dir)
-
-
-#-----------------------------------------------------------------------------
-# Command line options
-#-----------------------------------------------------------------------------
-
-class ClusterDirConfigLoader(BaseAppConfigLoader):
-
-    def _add_cluster_profile(self, parser):
-        paa = parser.add_argument
-        paa('-p', '--profile',
-            dest='Global.profile',type=unicode,
-            help=
-            """The string name of the profile to be used. This determines the name
-            of the cluster dir as: cluster_<profile>. The default profile is named
-            'default'.  The cluster directory is resolve this way if the
-            --cluster-dir option is not used.""",
-            metavar='Global.profile')
-
-    def _add_cluster_dir(self, parser):
-        paa = parser.add_argument
-        paa('--cluster-dir',
-            dest='Global.cluster_dir',type=unicode,
-            help="""Set the cluster dir. This overrides the logic used by the
-            --profile option.""",
-            metavar='Global.cluster_dir')
-
-    def _add_work_dir(self, parser):
-        paa = parser.add_argument
-        paa('--work-dir',
-            dest='Global.work_dir',type=unicode,
-            help='Set the working dir for the process.',
-            metavar='Global.work_dir')
-
-    def _add_clean_logs(self, parser):
-        paa = parser.add_argument
-        paa('--clean-logs',
-            dest='Global.clean_logs', action='store_true',
-            help='Delete old log flies before starting.')
-
-    def _add_no_clean_logs(self, parser):
-        paa = parser.add_argument
-        paa('--no-clean-logs',
-            dest='Global.clean_logs', action='store_false',
-            help="Don't Delete old log flies before starting.")
-
-    def _add_arguments(self):
-        super(ClusterDirConfigLoader, self)._add_arguments()
-        self._add_cluster_profile(self.parser)
-        self._add_cluster_dir(self.parser)
-        self._add_work_dir(self.parser)
-        self._add_clean_logs(self.parser)
-        self._add_no_clean_logs(self.parser)
-
-
-#-----------------------------------------------------------------------------
-# Crash handler for this application
-#-----------------------------------------------------------------------------
-
-
-_message_template = """\
-Oops, $self.app_name crashed. We do our best to make it stable, but...
-
-A crash report was automatically generated with the following information:
-  - A verbatim copy of the crash traceback.
-  - Data on your current $self.app_name configuration.
-
-It was left in the file named:
-\t'$self.crash_report_fname'
-If you can email this file to the developers, the information in it will help
-them in understanding and correcting the problem.
-
-You can mail it to: $self.contact_name at $self.contact_email
-with the subject '$self.app_name Crash Report'.
-
-If you want to do it now, the following command will work (under Unix):
-mail -s '$self.app_name Crash Report' $self.contact_email < $self.crash_report_fname
-
-To ensure accurate tracking of this issue, please file a report about it at:
-$self.bug_tracker
-"""
-
-class ClusterDirCrashHandler(CrashHandler):
-    """sys.excepthook for IPython itself, leaves a detailed report on disk."""
-
-    message_template = _message_template
-
-    def __init__(self, app):
-        contact_name = release.authors['Brian'][0]
-        contact_email = release.authors['Brian'][1]
-        bug_tracker = 'http://github.com/ipython/ipython/issues'
-        super(ClusterDirCrashHandler,self).__init__(
-            app, contact_name, contact_email, bug_tracker
-        )
-
-
-#-----------------------------------------------------------------------------
-# Main application
-#-----------------------------------------------------------------------------
-
-class ApplicationWithClusterDir(Application):
-    """An application that puts everything into a cluster directory.
-
-    Instead of looking for things in the ipython_dir, this type of application
-    will use its own private directory called the "cluster directory"
-    for things like config files, log files, etc.
-
-    The cluster directory is resolved as follows:
-
-    * If the ``--cluster-dir`` option is given, it is used.
-    * If ``--cluster-dir`` is not given, the application directory is 
-      resolve using the profile name as ``cluster_<profile>``. The search 
-      path for this directory is then i) cwd if it is found there
-      and ii) in ipython_dir otherwise.
-
-    The config file for the application is to be put in the cluster
-    dir and named the value of the ``config_file_name`` class attribute.
-    """
-
-    command_line_loader = ClusterDirConfigLoader
-    crash_handler_class = ClusterDirCrashHandler
-    auto_create_cluster_dir = True
-    # temporarily override default_log_level to INFO
-    default_log_level = logging.INFO
-
-    def create_default_config(self):
-        super(ApplicationWithClusterDir, self).create_default_config()
-        self.default_config.Global.profile = u'default'
-        self.default_config.Global.cluster_dir = u''
-        self.default_config.Global.work_dir = os.getcwd()
-        self.default_config.Global.log_to_file = False
-        self.default_config.Global.log_url = None
-        self.default_config.Global.clean_logs = False
-
-    def find_resources(self):
-        """This resolves the cluster directory.
-
-        This tries to find the cluster directory and if successful, it will
-        have done:
-        * Sets ``self.cluster_dir_obj`` to the :class:`ClusterDir` object for 
-          the application.
-        * Sets ``self.cluster_dir`` attribute of the application and config
-          objects.
-
-        The algorithm used for this is as follows:
-        1. Try ``Global.cluster_dir``.
-        2. Try using ``Global.profile``.
-        3. If both of these fail and ``self.auto_create_cluster_dir`` is
-           ``True``, then create the new cluster dir in the IPython directory.
-        4. If all fails, then raise :class:`ClusterDirError`.
-        """
-
-        try:
-            cluster_dir = self.command_line_config.Global.cluster_dir
-        except AttributeError:
-            cluster_dir = self.default_config.Global.cluster_dir
-        cluster_dir = expand_path(cluster_dir)
-        try:
-            self.cluster_dir_obj = ClusterDir.find_cluster_dir(cluster_dir)
-        except ClusterDirError:
-            pass
-        else:
-            self.log.info('Using existing cluster dir: %s' % \
-                self.cluster_dir_obj.location
-            )
-            self.finish_cluster_dir()
-            return
-
-        try:
-            self.profile = self.command_line_config.Global.profile
-        except AttributeError:
-            self.profile = self.default_config.Global.profile
-        try:
-            self.cluster_dir_obj = ClusterDir.find_cluster_dir_by_profile(
-                self.ipython_dir, self.profile)
-        except ClusterDirError:
-            pass
-        else:
-            self.log.info('Using existing cluster dir: %s' % \
-                self.cluster_dir_obj.location
-            )
-            self.finish_cluster_dir()
-            return
-
-        if self.auto_create_cluster_dir:
-            self.cluster_dir_obj = ClusterDir.create_cluster_dir_by_profile(
-                self.ipython_dir, self.profile
-            )
-            self.log.info('Creating new cluster dir: %s' % \
-                self.cluster_dir_obj.location
-            )
-            self.finish_cluster_dir()
-        else:
-            raise ClusterDirError('Could not find a valid cluster directory.')
-
-    def finish_cluster_dir(self):
-        # Set the cluster directory
-        self.cluster_dir = self.cluster_dir_obj.location
-        
-        # These have to be set because they could be different from the one
-        # that we just computed.  Because command line has the highest
-        # priority, this will always end up in the master_config.
-        self.default_config.Global.cluster_dir = self.cluster_dir
-        self.command_line_config.Global.cluster_dir = self.cluster_dir
-
-    def find_config_file_name(self):
-        """Find the config file name for this application."""
-        # For this type of Application it should be set as a class attribute.
-        if not hasattr(self, 'default_config_file_name'):
-            self.log.critical("No config filename found")
-        else:
-            self.config_file_name = self.default_config_file_name
-
-    def find_config_file_paths(self):
-        # Set the search path to to the cluster directory. We should NOT
-        # include IPython.config.default here as the default config files
-        # are ALWAYS automatically moved to the cluster directory.
-        conf_dir = os.path.join(get_ipython_package_dir(), 'config', 'default')
-        self.config_file_paths = (self.cluster_dir,)
-
-    def pre_construct(self):
-        # The log and security dirs were set earlier, but here we put them
-        # into the config and log them.
-        config = self.master_config
-        sdir = self.cluster_dir_obj.security_dir
-        self.security_dir = config.Global.security_dir = sdir
-        ldir = self.cluster_dir_obj.log_dir
-        self.log_dir = config.Global.log_dir = ldir
-        pdir = self.cluster_dir_obj.pid_dir
-        self.pid_dir = config.Global.pid_dir = pdir
-        self.log.info("Cluster directory set to: %s" % self.cluster_dir)
-        config.Global.work_dir = unicode(expand_path(config.Global.work_dir))
-        # Change to the working directory. We do this just before construct
-        # is called so all the components there have the right working dir.
-        self.to_work_dir()
-
-    def to_work_dir(self):
-        wd = self.master_config.Global.work_dir
-        if unicode(wd) != unicode(os.getcwd()):
-            os.chdir(wd)
-            self.log.info("Changing to working dir: %s" % wd)
-
-    def start_logging(self):
-        # Remove old log files
-        if self.master_config.Global.clean_logs:
-            log_dir = self.master_config.Global.log_dir
-            for f in os.listdir(log_dir):
-                if re.match(r'%s-\d+\.(log|err|out)'%self.name,f):
-                # if f.startswith(self.name + u'-') and f.endswith('.log'):
-                    os.remove(os.path.join(log_dir, f))
-        # Start logging to the new log file
-        if self.master_config.Global.log_to_file:
-            log_filename = self.name + u'-' + str(os.getpid()) + u'.log'
-            logfile = os.path.join(self.log_dir, log_filename)
-            open_log_file = open(logfile, 'w')
-        elif self.master_config.Global.log_url:
-            open_log_file = None
-        else:
-            open_log_file = sys.stdout
-        if open_log_file is not None:
-            self.log.removeHandler(self._log_handler)
-            self._log_handler = logging.StreamHandler(open_log_file)
-            self._log_formatter = logging.Formatter("[%(name)s] %(message)s")
-            self._log_handler.setFormatter(self._log_formatter)
-            self.log.addHandler(self._log_handler)
-        # log.startLogging(open_log_file)
-
-    def write_pid_file(self, overwrite=False):
-        """Create a .pid file in the pid_dir with my pid.
-
-        This must be called after pre_construct, which sets `self.pid_dir`.
-        This raises :exc:`PIDFileError` if the pid file exists already.
-        """
-        pid_file = os.path.join(self.pid_dir, self.name + u'.pid')
-        if os.path.isfile(pid_file):
-            pid = self.get_pid_from_file()
-            if not overwrite:
-                raise PIDFileError(
-                    'The pid file [%s] already exists. \nThis could mean that this '
-                    'server is already running with [pid=%s].' % (pid_file, pid)
-                )
-        with open(pid_file, 'w') as f:
-            self.log.info("Creating pid file: %s" % pid_file)
-            f.write(repr(os.getpid())+'\n')
-
-    def remove_pid_file(self):
-        """Remove the pid file.
-
-        This should be called at shutdown by registering a callback with
-        :func:`reactor.addSystemEventTrigger`. This needs to return
-        ``None``.
-        """
-        pid_file = os.path.join(self.pid_dir, self.name + u'.pid')
-        if os.path.isfile(pid_file):
-            try:
-                self.log.info("Removing pid file: %s" % pid_file)
-                os.remove(pid_file)
-            except:
-                self.log.warn("Error removing the pid file: %s" % pid_file)
-
-    def get_pid_from_file(self):
-        """Get the pid from the pid file.
-
-        If the  pid file doesn't exist a :exc:`PIDFileError` is raised.
-        """
-        pid_file = os.path.join(self.pid_dir, self.name + u'.pid')
-        if os.path.isfile(pid_file):
-            with open(pid_file, 'r') as f:
-                pid = int(f.read().strip())
-                return pid
-        else:
-            raise PIDFileError('pid file not found: %s' % pid_file)
-    
-    def check_pid(self, pid):
-        if os.name == 'nt':
-            try:
-                import ctypes
-                # returns 0 if no such process (of ours) exists
-                # positive int otherwise
-                p = ctypes.windll.kernel32.OpenProcess(1,0,pid)
-            except Exception:
-                self.log.warn(
-                    "Could not determine whether pid %i is running via `OpenProcess`. "
-                    " Making the likely assumption that it is."%pid
-                )
-                return True
-            return bool(p)
-        else:
-            try:
-                p = Popen(['ps','x'], stdout=PIPE, stderr=PIPE)
-                output,_ = p.communicate()
-            except OSError:
-                self.log.warn(
-                    "Could not determine whether pid %i is running via `ps x`. "
-                    " Making the likely assumption that it is."%pid
-                )
-                return True
-            pids = map(int, re.findall(r'^\W*\d+', output, re.MULTILINE))
-            return pid in pids
-    
  No newline at end of file

IPython/parallel/controller/controller.py

@@ -1,116 +0,0 b''
-#!/usr/bin/env python
-"""The IPython Controller with 0MQ
-This is a collection of one Hub and several Schedulers.
-"""
-#-----------------------------------------------------------------------------
-#  Copyright (C) 2010  The IPython Development Team
-#
-#  Distributed under the terms of the BSD License.  The full license is in
-#  the file COPYING, distributed as part of this software.
-#-----------------------------------------------------------------------------
-
-#-----------------------------------------------------------------------------
-# Imports
-#-----------------------------------------------------------------------------
-from __future__ import print_function
-
-from multiprocessing import Process
-
-import zmq
-from zmq.devices import ProcessMonitoredQueue
-# internal:
-from IPython.utils.importstring import import_item
-from IPython.utils.traitlets import Int, CStr, Instance, List, Bool
-
-from IPython.parallel.util import signal_children
-from .hub import Hub, HubFactory
-from .scheduler import launch_scheduler
-
-#-----------------------------------------------------------------------------
-# Configurable
-#-----------------------------------------------------------------------------
-
-
-class ControllerFactory(HubFactory):
-    """Configurable for setting up a Hub and Schedulers."""
-    
-    usethreads = Bool(False, config=True)
-    
-    # internal
-    children = List()
-    mq_class = CStr('zmq.devices.ProcessMonitoredQueue')
-    
-    def _usethreads_changed(self, name, old, new):
-        self.mq_class = 'zmq.devices.%sMonitoredQueue'%('Thread' if new else 'Process')
-        
-    def __init__(self, **kwargs):
-        super(ControllerFactory, self).__init__(**kwargs)
-        self.subconstructors.append(self.construct_schedulers)
-    
-    def start(self):
-        super(ControllerFactory, self).start()
-        child_procs = []
-        for child in self.children:
-            child.start()
-            if isinstance(child, ProcessMonitoredQueue):
-                child_procs.append(child.launcher)
-            elif isinstance(child, Process):
-                child_procs.append(child)
-        if child_procs:
-            signal_children(child_procs)
-        
-    
-    def construct_schedulers(self):
-        children = self.children
-        mq = import_item(self.mq_class)
-        
-        # maybe_inproc = 'inproc://monitor' if self.usethreads else self.monitor_url
-        # IOPub relay (in a Process)
-        q = mq(zmq.PUB, zmq.SUB, zmq.PUB, 'N/A','iopub')
-        q.bind_in(self.client_info['iopub'])
-        q.bind_out(self.engine_info['iopub'])
-        q.setsockopt_out(zmq.SUBSCRIBE, '')
-        q.connect_mon(self.monitor_url)
-        q.daemon=True
-        children.append(q)
-
-        # Multiplexer Queue (in a Process)
-        q = mq(zmq.XREP, zmq.XREP, zmq.PUB, 'in', 'out')
-        q.bind_in(self.client_info['mux'])
-        q.setsockopt_in(zmq.IDENTITY, 'mux')
-        q.bind_out(self.engine_info['mux'])
-        q.connect_mon(self.monitor_url)
-        q.daemon=True
-        children.append(q)
-
-        # Control Queue (in a Process)
-        q = mq(zmq.XREP, zmq.XREP, zmq.PUB, 'incontrol', 'outcontrol')
-        q.bind_in(self.client_info['control'])
-        q.setsockopt_in(zmq.IDENTITY, 'control')
-        q.bind_out(self.engine_info['control'])
-        q.connect_mon(self.monitor_url)
-        q.daemon=True
-        children.append(q)
-        # Task Queue (in a Process)
-        if self.scheme == 'pure':
-            self.log.warn("task::using pure XREQ Task scheduler")
-            q = mq(zmq.XREP, zmq.XREQ, zmq.PUB, 'intask', 'outtask')
-            q.bind_in(self.client_info['task'][1])
-            q.setsockopt_in(zmq.IDENTITY, 'task')
-            q.bind_out(self.engine_info['task'])
-            q.connect_mon(self.monitor_url)
-            q.daemon=True
-            children.append(q)
-        elif self.scheme == 'none':
-            self.log.warn("task::using no Task scheduler")
-            
-        else:
-            self.log.info("task::using Python %s Task scheduler"%self.scheme)
-            sargs = (self.client_info['task'][1], self.engine_info['task'],
-                                self.monitor_url, self.client_info['notification'])
-            kwargs = dict(scheme=self.scheme,logname=self.log.name, loglevel=self.log.level,
-                                                            config=dict(self.config))
-            q = Process(target=launch_scheduler, args=sargs, kwargs=kwargs)
-            q.daemon=True
-            children.append(q)
-

IPython/parallel/streamsession.py

@@ -1,419 +0,0 b''
-#!/usr/bin/env python
-"""edited session.py to work with streams, and move msg_type to the header
-"""
-#-----------------------------------------------------------------------------
-#  Copyright (C) 2010-2011  The IPython Development Team
-#
-#  Distributed under the terms of the BSD License.  The full license is in
-#  the file COPYING, distributed as part of this software.
-#-----------------------------------------------------------------------------
-
-
-import os
-import pprint
-import uuid
-from datetime import datetime
-
-try:
-    import cPickle
-    pickle = cPickle
-except:
-    cPickle = None
-    import pickle
-
-import zmq
-from zmq.utils import jsonapi
-from zmq.eventloop.zmqstream import ZMQStream
-
-from .util import ISO8601
-
-def squash_unicode(obj):
-    """coerce unicode back to bytestrings."""
-    if isinstance(obj,dict):
-        for key in obj.keys():
-            obj[key] = squash_unicode(obj[key])
-            if isinstance(key, unicode):
-                obj[squash_unicode(key)] = obj.pop(key)
-    elif isinstance(obj, list):
-        for i,v in enumerate(obj):
-            obj[i] = squash_unicode(v)
-    elif isinstance(obj, unicode):
-        obj = obj.encode('utf8')
-    return obj
-
-def _date_default(obj):
-    if isinstance(obj, datetime):
-        return obj.strftime(ISO8601)
-    else:
-        raise TypeError("%r is not JSON serializable"%obj)
-
-_default_key = 'on_unknown' if jsonapi.jsonmod.__name__ == 'jsonlib' else 'default'
-json_packer = lambda obj: jsonapi.dumps(obj, **{_default_key:_date_default})
-json_unpacker = lambda s: squash_unicode(jsonapi.loads(s))
-
-pickle_packer = lambda o: pickle.dumps(o,-1)
-pickle_unpacker = pickle.loads
-
-default_packer = json_packer
-default_unpacker = json_unpacker
-
-
-DELIM="<IDS|MSG>"
-
-class Message(object):
-    """A simple message object that maps dict keys to attributes.
-
-    A Message can be created from a dict and a dict from a Message instance
-    simply by calling dict(msg_obj)."""
-    
-    def __init__(self, msg_dict):
-        dct = self.__dict__
-        for k, v in dict(msg_dict).iteritems():
-            if isinstance(v, dict):
-                v = Message(v)
-            dct[k] = v
-
-    # Having this iterator lets dict(msg_obj) work out of the box.
-    def __iter__(self):
-        return iter(self.__dict__.iteritems())
-    
-    def __repr__(self):
-        return repr(self.__dict__)
-
-    def __str__(self):
-        return pprint.pformat(self.__dict__)
-
-    def __contains__(self, k):
-        return k in self.__dict__
-
-    def __getitem__(self, k):
-        return self.__dict__[k]
-
-
-def msg_header(msg_id, msg_type, username, session):
-    date=datetime.now().strftime(ISO8601)
-    return locals()
-
-def extract_header(msg_or_header):
-    """Given a message or header, return the header."""
-    if not msg_or_header:
-        return {}
-    try:
-        # See if msg_or_header is the entire message.
-        h = msg_or_header['header']
-    except KeyError:
-        try:
-            # See if msg_or_header is just the header
-            h = msg_or_header['msg_id']
-        except KeyError:
-            raise
-        else:
-            h = msg_or_header
-    if not isinstance(h, dict):
-        h = dict(h)
-    return h
-
-class StreamSession(object):
-    """tweaked version of IPython.zmq.session.Session, for development in Parallel"""
-    debug=False
-    key=None
-    
-    def __init__(self, username=None, session=None, packer=None, unpacker=None, key=None, keyfile=None):
-        if username is None:
-            username = os.environ.get('USER','username')
-        self.username = username
-        if session is None:
-            self.session = str(uuid.uuid4())
-        else:
-            self.session = session
-        self.msg_id = str(uuid.uuid4())
-        if packer is None:
-            self.pack = default_packer
-        else:
-            if not callable(packer):
-                raise TypeError("packer must be callable, not %s"%type(packer))
-            self.pack = packer
-        
-        if unpacker is None:
-            self.unpack = default_unpacker
-        else:
-            if not callable(unpacker):
-                raise TypeError("unpacker must be callable, not %s"%type(unpacker))
-            self.unpack = unpacker
-        
-        if key is not None and keyfile is not None:
-            raise TypeError("Must specify key OR keyfile, not both")
-        if keyfile is not None:
-            with open(keyfile) as f:
-                self.key = f.read().strip()
-        else:
-            self.key = key
-        if isinstance(self.key, unicode):
-            self.key = self.key.encode('utf8')
-        # print key, keyfile, self.key
-        self.none = self.pack({})
-            
-    def msg_header(self, msg_type):
-        h = msg_header(self.msg_id, msg_type, self.username, self.session)
-        self.msg_id = str(uuid.uuid4())
-        return h
-
-    def msg(self, msg_type, content=None, parent=None, subheader=None):
-        msg = {}
-        msg['header'] = self.msg_header(msg_type)
-        msg['msg_id'] = msg['header']['msg_id']
-        msg['parent_header'] = {} if parent is None else extract_header(parent)
-        msg['msg_type'] = msg_type
-        msg['content'] = {} if content is None else content
-        sub = {} if subheader is None else subheader
-        msg['header'].update(sub)
-        return msg
-
-    def check_key(self, msg_or_header):
-        """Check that a message's header has the right key"""
-        if self.key is None:
-            return True
-        header = extract_header(msg_or_header)
-        return header.get('key', None) == self.key
-            
-
-    def serialize(self, msg, ident=None):
-        content = msg.get('content', {})
-        if content is None:
-            content = self.none
-        elif isinstance(content, dict):
-            content = self.pack(content)
-        elif isinstance(content, bytes):
-            # content is already packed, as in a relayed message
-            pass
-        elif isinstance(content, unicode):
-            # should be bytes, but JSON often spits out unicode
-            content = content.encode('utf8')
-        else:
-            raise TypeError("Content incorrect type: %s"%type(content))
-
-        to_send = []
-
-        if isinstance(ident, list):
-            # accept list of idents
-            to_send.extend(ident)
-        elif ident is not None:
-            to_send.append(ident)
-        to_send.append(DELIM)
-        if self.key is not None:
-            to_send.append(self.key)
-        to_send.append(self.pack(msg['header']))
-        to_send.append(self.pack(msg['parent_header']))
-        to_send.append(content)
-
-        return to_send
-        
-    def send(self, stream, msg_or_type, content=None, buffers=None, parent=None, subheader=None, ident=None, track=False):
-        """Build and send a message via stream or socket.
-        
-        Parameters
-        ----------
-        
-        stream : zmq.Socket or ZMQStream
-            the socket-like object used to send the data
-        msg_or_type : str or Message/dict
-            Normally, msg_or_type will be a msg_type unless a message is being sent more
-            than once.
-        
-        content : dict or None
-            the content of the message (ignored if msg_or_type is a message)
-        buffers : list or None
-            the already-serialized buffers to be appended to the message
-        parent : Message or dict or None
-            the parent or parent header describing the parent of this message
-        subheader : dict or None
-            extra header keys for this message's header
-        ident : bytes or list of bytes
-            the zmq.IDENTITY routing path
-        track : bool
-            whether to track.  Only for use with Sockets, because ZMQStream objects cannot track messages.
-        
-        Returns
-        -------
-        msg : message dict
-            the constructed message
-        (msg,tracker) : (message dict, MessageTracker)
-            if track=True, then a 2-tuple will be returned, the first element being the constructed
-            message, and the second being the MessageTracker
-            
-        """
-
-        if not isinstance(stream, (zmq.Socket, ZMQStream)):
-            raise TypeError("stream must be Socket or ZMQStream, not %r"%type(stream))
-        elif track and isinstance(stream, ZMQStream):
-            raise TypeError("ZMQStream cannot track messages")
-        
-        if isinstance(msg_or_type, (Message, dict)):
-            # we got a Message, not a msg_type
-            # don't build a new Message
-            msg = msg_or_type
-        else:
-            msg = self.msg(msg_or_type, content, parent, subheader)
-        
-        buffers = [] if buffers is None else buffers
-        to_send = self.serialize(msg, ident)
-        flag = 0
-        if buffers:
-            flag = zmq.SNDMORE
-            _track = False
-        else:
-            _track=track
-        if track:
-            tracker = stream.send_multipart(to_send, flag, copy=False, track=_track)
-        else:
-            tracker = stream.send_multipart(to_send, flag, copy=False)
-        for b in buffers[:-1]:
-            stream.send(b, flag, copy=False)
-        if buffers:
-            if track:
-                tracker = stream.send(buffers[-1], copy=False, track=track)
-            else:
-                tracker = stream.send(buffers[-1], copy=False)
-                
-        # omsg = Message(msg)
-        if self.debug:
-            pprint.pprint(msg)
-            pprint.pprint(to_send)
-            pprint.pprint(buffers)
-        
-        msg['tracker'] = tracker
-        
-        return msg
-    
-    def send_raw(self, stream, msg, flags=0, copy=True, ident=None):
-        """Send a raw message via ident path.
-        
-        Parameters
-        ----------
-        msg : list of sendable buffers"""
-        to_send = []
-        if isinstance(ident, bytes):
-            ident = [ident]
-        if ident is not None:
-            to_send.extend(ident)
-        to_send.append(DELIM)
-        if self.key is not None:
-            to_send.append(self.key)
-        to_send.extend(msg)
-        stream.send_multipart(msg, flags, copy=copy)
-    
-    def recv(self, socket, mode=zmq.NOBLOCK, content=True, copy=True):
-        """receives and unpacks a message
-        returns [idents], msg"""
-        if isinstance(socket, ZMQStream):
-            socket = socket.socket
-        try:
-            msg = socket.recv_multipart(mode, copy=copy)
-        except zmq.ZMQError as e:
-            if e.errno == zmq.EAGAIN:
-                # We can convert EAGAIN to None as we know in this case
-                # recv_multipart won't return None.
-                return None
-            else:
-                raise
-        # return an actual Message object
-        # determine the number of idents by trying to unpack them.
-        # this is terrible:
-        idents, msg = self.feed_identities(msg, copy)
-        try:
-            return idents, self.unpack_message(msg, content=content, copy=copy)
-        except Exception as e:
-            print (idents, msg)
-            # TODO: handle it
-            raise e
-    
-    def feed_identities(self, msg, copy=True):
-        """feed until DELIM is reached, then return the prefix as idents and remainder as
-        msg. This is easily broken by setting an IDENT to DELIM, but that would be silly.
-        
-        Parameters
-        ----------
-        msg : a list of Message or bytes objects
-            the message to be split
-        copy : bool
-            flag determining whether the arguments are bytes or Messages
-        
-        Returns
-        -------
-        (idents,msg) : two lists
-            idents will always be a list of bytes - the indentity prefix
-            msg will be a list of bytes or Messages, unchanged from input
-            msg should be unpackable via self.unpack_message at this point.
-        """
-        ikey = int(self.key is not None)
-        minlen = 3 + ikey
-        msg = list(msg)
-        idents = []
-        while len(msg) > minlen:
-            if copy:
-                s = msg[0]
-            else:
-                s = msg[0].bytes
-            if s == DELIM:
-                msg.pop(0)
-                break
-            else:
-                idents.append(s)
-                msg.pop(0)
-                
-        return idents, msg
-    
-    def unpack_message(self, msg, content=True, copy=True):
-        """Return a message object from the format
-        sent by self.send.
-        
-        Parameters:
-        -----------
-        
-        content : bool (True)
-            whether to unpack the content dict (True), 
-            or leave it serialized (False)
-        
-        copy : bool (True)
-            whether to return the bytes (True), 
-            or the non-copying Message object in each place (False)
-        
-        """
-        ikey = int(self.key is not None)
-        minlen = 3 + ikey
-        message = {}
-        if not copy:
-            for i in range(minlen):
-                msg[i] = msg[i].bytes
-        if ikey:
-            if not self.key == msg[0]:
-                raise KeyError("Invalid Session Key: %s"%msg[0])
-        if not len(msg) >= minlen:
-            raise TypeError("malformed message, must have at least %i elements"%minlen)
-        message['header'] = self.unpack(msg[ikey+0])
-        message['msg_type'] = message['header']['msg_type']
-        message['parent_header'] = self.unpack(msg[ikey+1])
-        if content:
-            message['content'] = self.unpack(msg[ikey+2])
-        else:
-            message['content'] = msg[ikey+2]
-        
-        message['buffers'] = msg[ikey+3:]# [ m.buffer for m in msg[3:] ]
-        return message
-        
-
-def test_msg2obj():
-    am = dict(x=1)
-    ao = Message(am)
-    assert ao.x == am['x']
-
-    am['y'] = dict(z=1)
-    ao = Message(am)
-    assert ao.y.z == am['y']['z']
-    
-    k1, k2 = 'y', 'z'
-    assert ao[k1][k2] == am[k1][k2]
-    
-    am2 = dict(ao)
-    assert am['x'] == am2['x']
-    assert am['y']['z'] == am2['y']['z']

IPython/scripts/ipython-qtconsole

@@ -1,6 +0,0 b''
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from IPython.frontend.qt.console.ipythonqt import main
-
-main()