Various fixes to ipapp/ipcluster/ipengine/ipcontroller.

* Fixed bugs in handling of ``reuse_furls`` and ``secure``.
* Fixed ``--no-sep`` option.
* Refactored how the :attr:`Application.config_file_name` is handled
  to make it clearer and more robust.
* Tested all the apps.

IPython/core/application.py

@@ -1,449 +1,453 b''
 # encoding: utf-8
 """
 An application for IPython.
 
 All top-level applications should use the classes in this module for
 handling configuration and creating componenets.
 
 The job of an :class:`Application` is to create the master configuration 
 object and then create the components, passing the config to them.
 
 Authors:
 
 * Brian Granger
 * Fernando Perez
 
 Notes
 -----
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import logging
 import os
 import sys
 
 from IPython.core import release, crashhandler
 from IPython.utils.path import get_ipython_dir, get_ipython_package_dir
 from IPython.config.loader import (
     PyFileConfigLoader,
     ArgParseConfigLoader,
     Config,
 )
 
 #-----------------------------------------------------------------------------
 # Classes and functions
 #-----------------------------------------------------------------------------
 
 class ApplicationError(Exception):
     pass
 
 
 class BaseAppConfigLoader(ArgParseConfigLoader):
     """Default command line options for IPython based applications."""
 
     def _add_ipython_dir(self, parser):
         """Add the --ipython-dir option to the parser."""
         paa = parser.add_argument
         paa('--ipython-dir', 
             dest='Global.ipython_dir',type=unicode,
             help=
             """Set to override default location of the IPython directory
             IPYTHON_DIR, stored as Global.ipython_dir.  This can also be 
             specified through the environment variable IPYTHON_DIR.""",
             metavar='Global.ipython_dir')
 
     def _add_log_level(self, parser):
         """Add the --log-level option to the parser."""
         paa = parser.add_argument
         paa('--log-level',
             dest="Global.log_level",type=int,
             help='Set the log level (0,10,20,30,40,50).  Default is 30.',
             metavar='Global.log_level')
 
     def _add_arguments(self):
         self._add_ipython_dir(self.parser)
         self._add_log_level(self.parser)
 
 
 class Application(object):
     """Load a config, construct components and set them running.
 
     The configuration of an application can be done via three different Config
     objects, which are loaded and ultimately merged into a single one used
     from that point on by the app. These are:
 
     1. default_config: internal defaults, implemented in code.
     2. file_config: read from the filesystem.
     3. command_line_config: read from the system's command line flags.
 
     During initialization, 3 is actually read before 2, since at the
     command-line one may override the location of the file to be read.  But the
     above is the order in which the merge is made.
     """
 
     name = u'ipython'
     description = 'IPython: an enhanced interactive Python shell.'
     #: Usage message printed by argparse. If None, auto-generate
     usage = None
     #: The command line config loader.  Subclass of ArgParseConfigLoader.
     command_line_loader = BaseAppConfigLoader
-    #: The name of the config file to load.
-    config_file_name = u'ipython_config.py'
+    #: The name of the config file to load, determined at runtime
+    config_file_name = None
     #: The name of the default config file. Track separately from the actual
     #: name because some logic happens only if we aren't using the default.
-    default_config_file_name = config_file_name
+    default_config_file_name = u'ipython_config.py'
     default_log_level = logging.WARN
     #: Set by --profile option
     profile_name = None
     #: User's ipython directory, typically ~/.ipython/
     ipython_dir = None
     #: Internal defaults, implemented in code.
     default_config = None
     #: Read from the filesystem.
     file_config = None
     #: Read from the system's command line flags.
     command_line_config = None
     #: The final config that will be passed to the component.
     master_config = None
     #: A reference to the argv to be used (typically ends up being sys.argv[1:])
     argv = None
     #: extra arguments computed by the command-line loader
     extra_args = None
     #: The class to use as the crash handler.
     crash_handler_class = crashhandler.CrashHandler
 
     # Private attributes
     _exiting = False
     _initialized = False
 
     def __init__(self, argv=None):
         self.argv = sys.argv[1:] if argv is None else argv
         self.init_logger()
 
     def init_logger(self):
         self.log = logging.getLogger(self.__class__.__name__)
         # This is used as the default until the command line arguments are read.
         self.log.setLevel(self.default_log_level)
         self._log_handler = logging.StreamHandler()
         self._log_formatter = logging.Formatter("[%(name)s] %(message)s")
         self._log_handler.setFormatter(self._log_formatter)
         self.log.addHandler(self._log_handler)
 
     def _set_log_level(self, level):
         self.log.setLevel(level)
 
     def _get_log_level(self):
         return self.log.level
 
     log_level = property(_get_log_level, _set_log_level)
 
     def initialize(self):
         """Initialize the application.
 
         Loads all configuration information and sets all application state, but
         does not start any relevant processing (typically some kind of event
         loop).
 
         Once this method has been called, the application is flagged as
         initialized and the method becomes a no-op."""
         
         if self._initialized:
             return
 
         # The first part is protected with an 'attempt' wrapper, that will log
         # failures with the basic system traceback machinery.  Once our crash
         # handler is in place, we can let any subsequent exception propagate,
         # as our handler will log it with much better detail than the default.
         self.attempt(self.create_crash_handler)
 
         # Configuration phase
         # Default config (internally hardwired in application code)
         self.create_default_config()
         self.log_default_config()
         self.set_default_config_log_level()
 
         # Command-line config
         self.pre_load_command_line_config()
         self.load_command_line_config()
         self.set_command_line_config_log_level()
         self.post_load_command_line_config()
         self.log_command_line_config()
 
         # Find resources needed for filesystem access, using information from
         # the above two
         self.find_ipython_dir()
         self.find_resources()
         self.find_config_file_name()
         self.find_config_file_paths()
 
         # File-based config
         self.pre_load_file_config()
         self.load_file_config()
         self.set_file_config_log_level()
         self.post_load_file_config()
         self.log_file_config()
 
         # Merge all config objects into a single one the app can then use
         self.merge_configs()
         self.log_master_config()
 
         # Construction phase
         self.pre_construct()
         self.construct()
         self.post_construct()
 
         # Done, flag as such and
         self._initialized = True
 
     def start(self):
         """Start the application."""
         self.initialize()
         self.start_app()
 
     #-------------------------------------------------------------------------
     # Various stages of Application creation
     #-------------------------------------------------------------------------
 
     def create_crash_handler(self):
         """Create a crash handler, typically setting sys.excepthook to it."""
         self.crash_handler = self.crash_handler_class(self)
         sys.excepthook = self.crash_handler
 
     def create_default_config(self):
         """Create defaults that can't be set elsewhere.
 
         For the most part, we try to set default in the class attributes
         of Components.  But, defaults the top-level Application (which is
         not a HasTraitlets or Component) are not set in this way.  Instead
         we set them here.  The Global section is for variables like this that
         don't belong to a particular component.
         """
         c = Config()
         c.Global.ipython_dir = get_ipython_dir()
         c.Global.log_level = self.log_level
         self.default_config = c
 
     def log_default_config(self):
         self.log.debug('Default config loaded:')
         self.log.debug(repr(self.default_config))
 
     def set_default_config_log_level(self):
         try:
             self.log_level = self.default_config.Global.log_level
         except AttributeError:
             # Fallback to the default_log_level class attribute
             pass
 
     def create_command_line_config(self):
         """Create and return a command line config loader."""
         return self.command_line_loader(
             self.argv,
             description=self.description, 
             version=release.version,
             usage=self.usage
         )
 
     def pre_load_command_line_config(self):
         """Do actions just before loading the command line config."""
         pass
 
     def load_command_line_config(self):
         """Load the command line config."""
         loader = self.create_command_line_config()
         self.command_line_config = loader.load_config()
         self.extra_args = loader.get_extra_args()
 
     def set_command_line_config_log_level(self):
         try:
             self.log_level = self.command_line_config.Global.log_level
         except AttributeError:
             pass
 
     def post_load_command_line_config(self):
         """Do actions just after loading the command line config."""
         pass
 
     def log_command_line_config(self):
         self.log.debug("Command line config loaded:")
         self.log.debug(repr(self.command_line_config))
 
     def find_ipython_dir(self):
         """Set the IPython directory.
 
         This sets ``self.ipython_dir``, but the actual value that is passed to
         the application is kept in either ``self.default_config`` or
         ``self.command_line_config``.  This also adds ``self.ipython_dir`` to
         ``sys.path`` so config files there can be referenced by other config
         files.
         """
 
         try:
             self.ipython_dir = self.command_line_config.Global.ipython_dir
         except AttributeError:
             self.ipython_dir = self.default_config.Global.ipython_dir
         sys.path.append(os.path.abspath(self.ipython_dir))
         if not os.path.isdir(self.ipython_dir):
             os.makedirs(self.ipython_dir, mode=0777)
         self.log.debug("IPYTHON_DIR set to: %s" % self.ipython_dir)
 
     def find_resources(self):
         """Find other resources that need to be in place.
 
         Things like cluster directories need to be in place to find the
         config file.  These happen right after the IPython directory has
         been set.
         """
         pass
 
     def find_config_file_name(self):
         """Find the config file name for this application.
 
         This must set ``self.config_file_name`` to the filename of the
         config file to use (just the filename). The search paths for the
         config file are set in :meth:`find_config_file_paths` and then passed
         to the config file loader where they are resolved to an absolute path.
 
         If a profile has been set at the command line, this will resolve it.
         """
-
         try:
             self.config_file_name = self.command_line_config.Global.config_file
         except AttributeError:
             pass
+        else:
+            return
 
         try:
             self.profile_name = self.command_line_config.Global.profile
         except AttributeError:
-            pass
+            # Just use the default as there is no profile
+            self.config_file_name = self.default_config_file_name
         else:
-            name_parts = self.config_file_name.split('.')
+            # Use the default config file name and profile name if set
+            # to determine the used config file name.
+            name_parts = self.default_config_file_name.split('.')
             name_parts.insert(1, u'_' + self.profile_name + u'.')
             self.config_file_name = ''.join(name_parts)
 
     def find_config_file_paths(self):
         """Set the search paths for resolving the config file.
 
         This must set ``self.config_file_paths`` to a sequence of search
         paths to pass to the config file loader.
         """
         # Include our own profiles directory last, so that users can still find
         # our shipped copies of builtin profiles even if they don't have them
         # in their local ipython directory.
         prof_dir = os.path.join(get_ipython_package_dir(), 'config', 'profile')
         self.config_file_paths = (os.getcwd(), self.ipython_dir, prof_dir)
 
     def pre_load_file_config(self):
         """Do actions before the config file is loaded."""
         pass
 
     def load_file_config(self):
         """Load the config file.
         
         This tries to load the config file from disk.  If successful, the
         ``CONFIG_FILE`` config variable is set to the resolved config file
         location.  If not successful, an empty config is used.
         """
         self.log.debug("Attempting to load config file: %s" %
                        self.config_file_name)
         loader = PyFileConfigLoader(self.config_file_name,
                                     path=self.config_file_paths)
         try:
             self.file_config = loader.load_config()
             self.file_config.Global.config_file = loader.full_filename
         except IOError:
             # Only warn if the default config file was NOT being used.
             if not self.config_file_name==self.default_config_file_name:
                 self.log.warn("Config file not found, skipping: %s" %
                                self.config_file_name, exc_info=True)
             self.file_config = Config()
         except:
             self.log.warn("Error loading config file: %s" %
                           self.config_file_name, exc_info=True)
             self.file_config = Config()
 
     def set_file_config_log_level(self):
         # We need to keeep self.log_level updated.  But we only use the value
         # of the file_config if a value was not specified at the command
         # line, because the command line overrides everything.
         if not hasattr(self.command_line_config.Global, 'log_level'):
             try:
                 self.log_level = self.file_config.Global.log_level
             except AttributeError:
                 pass # Use existing value
 
     def post_load_file_config(self):
         """Do actions after the config file is loaded."""
         pass
 
     def log_file_config(self):
         if hasattr(self.file_config.Global, 'config_file'):
             self.log.debug("Config file loaded: %s" %
                            self.file_config.Global.config_file)
             self.log.debug(repr(self.file_config))
 
     def merge_configs(self):
         """Merge the default, command line and file config objects."""
         config = Config()
         config._merge(self.default_config)
         config._merge(self.file_config)
         config._merge(self.command_line_config)
 
         # XXX fperez - propose to Brian we rename master_config to simply
         # config, I think this is going to be heavily used in examples and
         # application code and the name is shorter/easier to find/remember.
         # For now, just alias it...
         self.master_config = config
         self.config = config
 
     def log_master_config(self):
         self.log.debug("Master config created:")
         self.log.debug(repr(self.master_config))
 
     def pre_construct(self):
         """Do actions after the config has been built, but before construct."""
         pass
 
     def construct(self):
         """Construct the main components that make up this app."""
         self.log.debug("Constructing components for application")
 
     def post_construct(self):
         """Do actions after construct, but before starting the app."""
         pass
 
     def start_app(self):
         """Actually start the app."""
         self.log.debug("Starting application")
 
     #-------------------------------------------------------------------------
     # Utility methods
     #-------------------------------------------------------------------------
 
     def exit(self, exit_status=0):
         if self._exiting:
             pass
         else:
             self.log.debug("Exiting application: %s" % self.name)
             self._exiting = True
             sys.exit(exit_status)
 
     def attempt(self, func):
         try:
             func()
         except SystemExit:
             raise
         except:
             self.log.critical("Aborting application: %s" % self.name,
                               exc_info=True)
             self.exit(0)
 

IPython/core/ipapp.py

@@ -1,654 +1,653 b''
 #!/usr/bin/env python
 # encoding: utf-8
 """
 The :class:`~IPython.core.application.Application` object for the command
 line :command:`ipython` program.
 
 Authors
 -------
 
 * Brian Granger
 * Fernando Perez
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2010  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 from __future__ import absolute_import
 
 import logging
 import os
 import sys
 
 from IPython.core import release
 from IPython.core.crashhandler import CrashHandler
 from IPython.core.application import Application, BaseAppConfigLoader
 from IPython.core.iplib import InteractiveShell
 from IPython.config.loader import (
     Config,
     PyFileConfigLoader
 )
 from IPython.lib import inputhook
 from IPython.utils.path import filefind, get_ipython_dir
 from . import usage
 
 #-----------------------------------------------------------------------------
 # Globals, utilities and helpers
 #-----------------------------------------------------------------------------
 
 #: The default config file name for this application.
 default_config_file_name = u'ipython_config.py'
 
 
 class IPAppConfigLoader(BaseAppConfigLoader):
 
     def _add_arguments(self):
         super(IPAppConfigLoader, self)._add_arguments()
         paa = self.parser.add_argument
         paa('-p', 
             '--profile', dest='Global.profile', type=unicode,
             help=
             """The string name of the ipython profile to be used.  Assume that your
             config file is ipython_config-<name>.py (looks in current dir first,
             then in IPYTHON_DIR). This is a quick way to keep and load multiple
             config files for different tasks, especially if include your basic one
             in your more specialized ones.  You can keep a basic
             IPYTHON_DIR/ipython_config.py file and then have other 'profiles' which
             include this one and load extra things for particular tasks.""",
             metavar='Global.profile')
         paa('--config-file', 
             dest='Global.config_file', type=unicode,
             help=
             """Set the config file name to override default.  Normally IPython
             loads ipython_config.py (from current directory) or
             IPYTHON_DIR/ipython_config.py.  If the loading of your config file
             fails, IPython starts with a bare bones configuration (no modules
             loaded at all).""",
             metavar='Global.config_file')
         paa('--autocall', 
             dest='InteractiveShell.autocall', type=int, 
             help=
             """Make IPython automatically call any callable object even if you
             didn't type explicit parentheses. For example, 'str 43' becomes
             'str(43)' automatically.  The value can be '0' to disable the feature,
             '1' for 'smart' autocall, where it is not applied if there are no more
             arguments on the line, and '2' for 'full' autocall, where all callable
             objects are automatically called (even if no arguments are present).
             The default is '1'.""",
             metavar='InteractiveShell.autocall')
         paa('--autoindent',
             action='store_true', dest='InteractiveShell.autoindent',
             help='Turn on autoindenting.')
         paa('--no-autoindent',
             action='store_false', dest='InteractiveShell.autoindent',
             help='Turn off autoindenting.')
         paa('--automagic',
             action='store_true', dest='InteractiveShell.automagic',
             help=
             """Turn on the auto calling of magic commands. Type %%magic at the
             IPython  prompt  for  more information.""")
         paa('--no-automagic',
             action='store_false', dest='InteractiveShell.automagic',
             help='Turn off the auto calling of magic commands.')
         paa('--autoedit-syntax',
             action='store_true', dest='InteractiveShell.autoedit_syntax',
             help='Turn on auto editing of files with syntax errors.')
         paa('--no-autoedit-syntax',
             action='store_false', dest='InteractiveShell.autoedit_syntax',
             help='Turn off auto editing of files with syntax errors.')
         paa('--banner',
             action='store_true', dest='Global.display_banner',
             help='Display a banner upon starting IPython.')
         paa('--no-banner',
             action='store_false', dest='Global.display_banner',
             help="Don't display a banner upon starting IPython.")
         paa('--cache-size',
             type=int, dest='InteractiveShell.cache_size',
             help=
             """Set the size of the output cache.  The default is 1000, you can
             change it permanently in your config file.  Setting it to 0 completely
             disables the caching system, and the minimum value accepted is 20 (if
             you provide a value less than 20, it is reset to 0 and a warning is
             issued).  This limit is defined because otherwise you'll spend more
             time re-flushing a too small cache than working""",
             metavar='InteractiveShell.cache_size')
         paa('--classic',
             action='store_true', dest='Global.classic',
             help="Gives IPython a similar feel to the classic Python prompt.")
         paa('--colors',
             type=str, dest='InteractiveShell.colors',
             help="Set the color scheme (NoColor, Linux, and LightBG).",
             metavar='InteractiveShell.colors')
         paa('--color-info',
             action='store_true', dest='InteractiveShell.color_info',
             help=
             """IPython can display information about objects via a set of func-
             tions, and optionally can use colors for this, syntax highlighting
             source code and various other elements.  However, because this
             information is passed through a pager (like 'less') and many pagers get
             confused with color codes, this option is off by default.  You can test
             it and turn it on permanently in your ipython_config.py file if it
             works for you.  Test it and turn it on permanently if it works with
             your system.  The magic function %%color_info allows you to toggle this
             inter- actively for testing.""")
         paa('--no-color-info',
             action='store_false', dest='InteractiveShell.color_info',
             help="Disable using colors for info related things.")
         paa('--confirm-exit',
             action='store_true', dest='InteractiveShell.confirm_exit',
             help=
             """Set to confirm when you try to exit IPython with an EOF (Control-D
             in Unix, Control-Z/Enter in Windows). By typing 'exit', 'quit' or
             '%%Exit', you can force a direct exit without any confirmation.""")
         paa('--no-confirm-exit',
             action='store_false', dest='InteractiveShell.confirm_exit',
             help="Don't prompt the user when exiting.")
         paa('--deep-reload',
             action='store_true', dest='InteractiveShell.deep_reload',
             help=
             """Enable deep (recursive) reloading by default. IPython can use the
             deep_reload module which reloads changes in modules recursively (it
             replaces the reload() function, so you don't need to change anything to
             use it). deep_reload() forces a full reload of modules whose code may
             have changed, which the default reload() function does not.  When
             deep_reload is off, IPython will use the normal reload(), but
             deep_reload will still be available as dreload(). This fea- ture is off
             by default [which means that you have both normal reload() and
             dreload()].""")
         paa('--no-deep-reload',
             action='store_false', dest='InteractiveShell.deep_reload',
             help="Disable deep (recursive) reloading by default.")
         paa('--editor',
             type=str, dest='InteractiveShell.editor',
             help="Set the editor used by IPython (default to $EDITOR/vi/notepad).",
             metavar='InteractiveShell.editor')
         paa('--log','-l', 
             action='store_true', dest='InteractiveShell.logstart',
             help="Start logging to the default log file (./ipython_log.py).")
         paa('--logfile','-lf',
             type=unicode, dest='InteractiveShell.logfile',
             help="Start logging to logfile with this name.",
             metavar='InteractiveShell.logfile')
         paa('--log-append','-la',
             type=unicode, dest='InteractiveShell.logappend',
             help="Start logging to the given file in append mode.",
             metavar='InteractiveShell.logfile')
         paa('--pdb',
             action='store_true', dest='InteractiveShell.pdb',
             help="Enable auto calling the pdb debugger after every exception.")
         paa('--no-pdb',
             action='store_false', dest='InteractiveShell.pdb',
             help="Disable auto calling the pdb debugger after every exception.")
         paa('--pprint',
             action='store_true', dest='InteractiveShell.pprint',
             help="Enable auto pretty printing of results.")
         paa('--no-pprint',
             action='store_false', dest='InteractiveShell.pprint',
             help="Disable auto auto pretty printing of results.")
         paa('--prompt-in1','-pi1',
             type=str, dest='InteractiveShell.prompt_in1',
             help=
             """Set the main input prompt ('In [\#]: ').  Note that if you are using
             numbered prompts, the number is represented with a '\#' in the string.
             Don't forget to quote strings with spaces embedded in them. Most
             bash-like escapes can be used to customize IPython's prompts, as well
             as a few additional ones which are IPython-spe- cific.  All valid
             prompt escapes are described in detail in the Customization section of
             the IPython manual.""",
             metavar='InteractiveShell.prompt_in1')
         paa('--prompt-in2','-pi2',
             type=str, dest='InteractiveShell.prompt_in2',
             help=
             """Set the secondary input prompt (' .\D.: ').  Similar to the previous
             option, but used for the continuation prompts. The special sequence
             '\D' is similar to '\#', but with all digits replaced by dots (so you
             can have your continuation prompt aligned with your input prompt).
             Default: ' .\D.: ' (note three spaces at the start for alignment with
             'In [\#]')""",
             metavar='InteractiveShell.prompt_in2')
         paa('--prompt-out','-po',
             type=str, dest='InteractiveShell.prompt_out',
             help="Set the output prompt ('Out[\#]:')",
             metavar='InteractiveShell.prompt_out')
         paa('--quick',
             action='store_true', dest='Global.quick',
             help="Enable quick startup with no config files.")
         paa('--readline',
             action='store_true', dest='InteractiveShell.readline_use',
             help="Enable readline for command line usage.")
         paa('--no-readline',
             action='store_false', dest='InteractiveShell.readline_use',
             help="Disable readline for command line usage.")
         paa('--screen-length','-sl',
             type=int, dest='InteractiveShell.screen_length',
             help=
             """Number of lines of your screen, used to control printing of very
             long strings.  Strings longer than this number of lines will be sent
             through a pager instead of directly printed.  The default value for
             this is 0, which means IPython will auto-detect your screen size every
             time it needs to print certain potentially long strings (this doesn't
             change the behavior of the 'print' keyword, it's only triggered
             internally). If for some reason this isn't working well (it needs
             curses support), specify it yourself. Otherwise don't change the
             default.""",
             metavar='InteractiveShell.screen_length')
         paa('--separate-in','-si',
             type=str, dest='InteractiveShell.separate_in',
             help="Separator before input prompts.  Default '\\n'.",
             metavar='InteractiveShell.separate_in')
         paa('--separate-out','-so',
             type=str, dest='InteractiveShell.separate_out',
             help="Separator before output prompts.  Default 0 (nothing).",
             metavar='InteractiveShell.separate_out')
         paa('--separate-out2','-so2', 
             type=str, dest='InteractiveShell.separate_out2',
             help="Separator after output prompts.  Default 0 (nonight).",
             metavar='InteractiveShell.separate_out2')
-        paa('-no-sep',
+        paa('--no-sep',
             action='store_true', dest='Global.nosep',
             help="Eliminate all spacing between prompts.")
         paa('--term-title',
             action='store_true', dest='InteractiveShell.term_title',
             help="Enable auto setting the terminal title.")
         paa('--no-term-title',
             action='store_false', dest='InteractiveShell.term_title',
             help="Disable auto setting the terminal title.")
         paa('--xmode',
             type=str, dest='InteractiveShell.xmode',
             help=
             """Exception reporting mode ('Plain','Context','Verbose').  Plain:
             similar to python's normal traceback printing.  Context: prints 5 lines
             of context source code around each line in the traceback.  Verbose:
             similar to Context, but additionally prints the variables currently
             visible where the exception happened (shortening their strings if too
             long).  This can potentially be very slow, if you happen to have a huge
             data structure whose string representation is complex to compute.
             Your computer may appear to freeze for a while with cpu usage at 100%%.
             If this occurs, you can cancel the traceback with Ctrl-C (maybe hitting
             it more than once).
             """,
             metavar='InteractiveShell.xmode')
         paa('--ext',
             type=str, dest='Global.extra_extension',
             help="The dotted module name of an IPython extension to load.",
             metavar='Global.extra_extension')
         paa('-c',
             type=str, dest='Global.code_to_run',
             help="Execute the given command string.",
             metavar='Global.code_to_run')
         paa('-i',
             action='store_true', dest='Global.force_interact',
             help=
             "If running code from the command line, become interactive afterwards.")
 
         # Options to start with GUI control enabled from the beginning
         paa('--gui',
             type=str, dest='Global.gui',
             help="Enable GUI event loop integration ('qt', 'wx', 'gtk').",
             metavar='gui-mode')
         paa('--pylab','-pylab',
             type=str, dest='Global.pylab',
             nargs='?', const='auto', metavar='gui-mode',
             help="Pre-load matplotlib and numpy for interactive use. "+
             "If no value is given, the gui backend is matplotlib's, else use "+
             "one of:  ['tk', 'qt', 'wx', 'gtk'].")
 
         # Legacy GUI options.  Leave them in for backwards compatibility, but the
         # 'thread' names are really a misnomer now.
         paa('--wthread', '-wthread',
             action='store_true', dest='Global.wthread',
             help=
             """Enable wxPython event loop integration. (DEPRECATED, use --gui wx)""")
         paa('--q4thread', '--qthread', '-q4thread', '-qthread',
             action='store_true', dest='Global.q4thread',
             help=
             """Enable Qt4 event loop integration. Qt3 is no longer supported.
             (DEPRECATED, use --gui qt)""")
         paa('--gthread', '-gthread',
             action='store_true', dest='Global.gthread',
             help=
             """Enable GTK event loop integration. (DEPRECATED, use --gui gtk)""")
 
 
 #-----------------------------------------------------------------------------
 # Crash handler for this application
 #-----------------------------------------------------------------------------
 
 
 _message_template = """\
 Oops, $self.app_name crashed. We do our best to make it stable, but...
 
 A crash report was automatically generated with the following information:
   - A verbatim copy of the crash traceback.
   - A copy of your input history during this session.
   - Data on your current $self.app_name configuration.
 
 It was left in the file named:
 \t'$self.crash_report_fname'
 If you can email this file to the developers, the information in it will help
 them in understanding and correcting the problem.
 
 You can mail it to: $self.contact_name at $self.contact_email
 with the subject '$self.app_name Crash Report'.
 
 If you want to do it now, the following command will work (under Unix):
 mail -s '$self.app_name Crash Report' $self.contact_email < $self.crash_report_fname
 
 To ensure accurate tracking of this issue, please file a report about it at:
 $self.bug_tracker
 """
 
 class IPAppCrashHandler(CrashHandler):
     """sys.excepthook for IPython itself, leaves a detailed report on disk."""
 
     message_template = _message_template
 
     def __init__(self, app):
         contact_name = release.authors['Fernando'][0]
         contact_email = release.authors['Fernando'][1]
         bug_tracker = 'https://bugs.launchpad.net/ipython/+filebug'
         super(IPAppCrashHandler,self).__init__(
             app, contact_name, contact_email, bug_tracker
         )
 
     def make_report(self,traceback):
         """Return a string containing a crash report."""
 
         sec_sep = self.section_sep
         # Start with parent report
         report = [super(IPAppCrashHandler, self).make_report(traceback)]
         # Add interactive-specific info we may have
         rpt_add = report.append
         try:
             rpt_add(sec_sep+"History of session input:")
             for line in self.app.shell.user_ns['_ih']:
                 rpt_add(line)
             rpt_add('\n*** Last line of input (may not be in above history):\n')
             rpt_add(self.app.shell._last_input_line+'\n')
         except:
             pass
 
         return ''.join(report)
 
 
 #-----------------------------------------------------------------------------
 # Main classes and functions
 #-----------------------------------------------------------------------------
 
 class IPythonApp(Application):
     name = u'ipython'
     #: argparse formats better the 'usage' than the 'description' field
     description = None
     usage = usage.cl_usage
     command_line_loader = IPAppConfigLoader
-    config_file_name = default_config_file_name
+    default_config_file_name = default_config_file_name
     crash_handler_class = IPAppCrashHandler
 
     def create_default_config(self):
         super(IPythonApp, self).create_default_config()
         # Eliminate multiple lookups
         Global = self.default_config.Global
 
         # Set all default values
         Global.display_banner = True
         
         # If the -c flag is given or a file is given to run at the cmd line
         # like "ipython foo.py", normally we exit without starting the main
         # loop.  The force_interact config variable allows a user to override
         # this and interact.  It is also set by the -i cmd line flag, just
         # like Python.
         Global.force_interact = False
 
         # By default always interact by starting the IPython mainloop.
         Global.interact = True
 
         # No GUI integration by default
         Global.gui = False
         # Pylab off by default
         Global.pylab = False
 
         # Deprecated versions of gui support that used threading, we support
         # them just for bacwards compatibility as an alternate spelling for
         # '--gui X'
         Global.qthread = False
         Global.q4thread = False
         Global.wthread = False
         Global.gthread = False
 
     def load_file_config(self):
         if hasattr(self.command_line_config.Global, 'quick'):
             if self.command_line_config.Global.quick:
                 self.file_config = Config()
                 return
         super(IPythonApp, self).load_file_config()
 
     def post_load_file_config(self):
         if hasattr(self.command_line_config.Global, 'extra_extension'):
             if not hasattr(self.file_config.Global, 'extensions'):
                 self.file_config.Global.extensions = []
             self.file_config.Global.extensions.append(
                 self.command_line_config.Global.extra_extension)
             del self.command_line_config.Global.extra_extension
 
     def pre_construct(self):
         config = self.master_config
 
         if hasattr(config.Global, 'classic'):
             if config.Global.classic:
                 config.InteractiveShell.cache_size = 0
                 config.InteractiveShell.pprint = 0
                 config.InteractiveShell.prompt_in1 = '>>> '
                 config.InteractiveShell.prompt_in2 = '... '
                 config.InteractiveShell.prompt_out = ''
                 config.InteractiveShell.separate_in = \
                     config.InteractiveShell.separate_out = \
                     config.InteractiveShell.separate_out2 = ''
                 config.InteractiveShell.colors = 'NoColor'
                 config.InteractiveShell.xmode = 'Plain'
 
         if hasattr(config.Global, 'nosep'):
             if config.Global.nosep:
                 config.InteractiveShell.separate_in = \
                 config.InteractiveShell.separate_out = \
                 config.InteractiveShell.separate_out2 = ''
 
         # if there is code of files to run from the cmd line, don't interact
         # unless the -i flag (Global.force_interact) is true.
         code_to_run = config.Global.get('code_to_run','')
         file_to_run = False
         if self.extra_args and self.extra_args[0]:
                 file_to_run = True
         if file_to_run or code_to_run:
             if not config.Global.force_interact:
                 config.Global.interact = False
 
     def construct(self):
         # I am a little hesitant to put these into InteractiveShell itself.
         # But that might be the place for them
         sys.path.insert(0, '')
 
         # Create an InteractiveShell instance
         self.shell = InteractiveShell(None, self.master_config)
 
     def post_construct(self):
         """Do actions after construct, but before starting the app."""
         config = self.master_config
         
         # shell.display_banner should always be False for the terminal 
         # based app, because we call shell.show_banner() by hand below
         # so the banner shows *before* all extension loading stuff.
         self.shell.display_banner = False
-
         if config.Global.display_banner and \
             config.Global.interact:
             self.shell.show_banner()
 
         # Make sure there is a space below the banner.
         if self.log_level <= logging.INFO: print
 
         # Now a variety of things that happen after the banner is printed.
         self._enable_gui_pylab()
         self._load_extensions()
         self._run_exec_lines()
         self._run_exec_files()
         self._run_cmd_line_code()
 
     def _enable_gui_pylab(self):
         """Enable GUI event loop integration, taking pylab into account."""
         Global = self.master_config.Global
 
         # Select which gui to use
         if Global.gui:
             gui = Global.gui
         # The following are deprecated, but there's likely to be a lot of use
         # of this form out there, so we might as well support it for now.  But
         # the --gui option above takes precedence.
         elif Global.wthread:
             gui = inputhook.GUI_WX
         elif Global.qthread:
             gui = inputhook.GUI_QT
         elif Global.gthread:
             gui = inputhook.GUI_GTK
         else:
             gui = None
 
         # Using --pylab will also require gui activation, though which toolkit
         # to use may be chosen automatically based on mpl configuration.
         if Global.pylab:
             activate = self.shell.enable_pylab
             if Global.pylab == 'auto':
                 gui = None
             else:
                 gui = Global.pylab
         else:
             # Enable only GUI integration, no pylab
             activate = inputhook.enable_gui
 
         if gui or Global.pylab:
             try:
                 self.log.info("Enabling GUI event loop integration, "
                               "toolkit=%s, pylab=%s" % (gui, Global.pylab) )
                 activate(gui)
             except:
                 self.log.warn("Error in enabling GUI event loop integration:")
                 self.shell.showtraceback()
 
     def _load_extensions(self):
         """Load all IPython extensions in Global.extensions.
 
         This uses the :meth:`InteractiveShell.load_extensions` to load all
         the extensions listed in ``self.master_config.Global.extensions``.
         """
         try:
             if hasattr(self.master_config.Global, 'extensions'):
                 self.log.debug("Loading IPython extensions...")
                 extensions = self.master_config.Global.extensions
                 for ext in extensions:
                     try:
                         self.log.info("Loading IPython extension: %s" % ext)                        
                         self.shell.load_extension(ext)
                     except:
                         self.log.warn("Error in loading extension: %s" % ext)
                         self.shell.showtraceback()
         except:
             self.log.warn("Unknown error in loading extensions:")
             self.shell.showtraceback()
 
     def _run_exec_lines(self):
         """Run lines of code in Global.exec_lines in the user's namespace."""
         try:
             if hasattr(self.master_config.Global, 'exec_lines'):
                 self.log.debug("Running code from Global.exec_lines...")
                 exec_lines = self.master_config.Global.exec_lines
                 for line in exec_lines:
                     try:
                         self.log.info("Running code in user namespace: %s" % line)
                         self.shell.runlines(line)
                     except:
                         self.log.warn("Error in executing line in user namespace: %s" % line)
                         self.shell.showtraceback()
         except:
             self.log.warn("Unknown error in handling Global.exec_lines:")
             self.shell.showtraceback()
 
     def _exec_file(self, fname):
         full_filename = filefind(fname, [u'.', self.ipython_dir])
         if os.path.isfile(full_filename):
             if full_filename.endswith(u'.py'):
                 self.log.info("Running file in user namespace: %s" % full_filename)
                 self.shell.safe_execfile(full_filename, self.shell.user_ns)
             elif full_filename.endswith('.ipy'):
                 self.log.info("Running file in user namespace: %s" % full_filename)
                 self.shell.safe_execfile_ipy(full_filename)
             else:
                 self.log.warn("File does not have a .py or .ipy extension: <%s>" % full_filename)
 
     def _run_exec_files(self):
         try:
             if hasattr(self.master_config.Global, 'exec_files'):
                 self.log.debug("Running files in Global.exec_files...")
                 exec_files = self.master_config.Global.exec_files
                 for fname in exec_files:
                     self._exec_file(fname)
         except:
             self.log.warn("Unknown error in handling Global.exec_files:")
             self.shell.showtraceback()
 
     def _run_cmd_line_code(self):
         if hasattr(self.master_config.Global, 'code_to_run'):
             line = self.master_config.Global.code_to_run
             try:
                 self.log.info("Running code given at command line (-c): %s" % line)
                 self.shell.runlines(line)
             except:
                 self.log.warn("Error in executing line in user namespace: %s" % line)
                 self.shell.showtraceback()
             return
         # Like Python itself, ignore the second if the first of these is present
         try:
             fname = self.extra_args[0]
         except:
             pass
         else:
             try:
                 self._exec_file(fname)
             except:
                 self.log.warn("Error in executing file in user namespace: %s" % fname)
                 self.shell.showtraceback()
 
     def start_app(self):
         if self.master_config.Global.interact:
             self.log.debug("Starting IPython's mainloop...")
             self.shell.mainloop()
         else:
             self.log.debug("IPython not interactive, start_app is no-op...")
 
 
 def load_default_config(ipython_dir=None):
     """Load the default config file from the default ipython_dir.
 
     This is useful for embedded shells.
     """
     if ipython_dir is None:
         ipython_dir = get_ipython_dir()
     cl = PyFileConfigLoader(default_config_file_name, ipython_dir)
     config = cl.load_config()
     return config
 
 
 def launch_new_instance():
     """Create and run a full blown IPython instance"""
     app = IPythonApp()
     app.start()
 
 
 if __name__ == '__main__':
     launch_new_instance()

IPython/kernel/clusterdir.py

@@ -1,537 +1,539 b''
 #!/usr/bin/env python
 # encoding: utf-8
 """
 The IPython cluster directory
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 from __future__ import with_statement
 
 import os
 import shutil
 import sys
 import warnings
 
 from twisted.python import log
 
 from IPython.config.loader import PyFileConfigLoader
 from IPython.core.application import Application, BaseAppConfigLoader
 from IPython.core.component import Component
 from IPython.core.crashhandler import CrashHandler
 from IPython.core import release
 from IPython.utils.path import (
     get_ipython_package_dir,
     expand_path
 )
 from IPython.utils.traitlets import Unicode
 
 #-----------------------------------------------------------------------------
 # Warnings control
 #-----------------------------------------------------------------------------
 # Twisted generates annoying warnings with Python 2.6, as will do other code
 # that imports 'sets' as of today
 warnings.filterwarnings('ignore', 'the sets module is deprecated',
                         DeprecationWarning )
 
 # This one also comes from Twisted
 warnings.filterwarnings('ignore', 'the sha module is deprecated',
                         DeprecationWarning)
 
 #-----------------------------------------------------------------------------
 # Module errors
 #-----------------------------------------------------------------------------
 
 class ClusterDirError(Exception):
     pass
 
 
 class PIDFileError(Exception):
     pass
 
 
 #-----------------------------------------------------------------------------
 # Class for managing cluster directories
 #-----------------------------------------------------------------------------
 
 class ClusterDir(Component):
     """An object to manage the cluster directory and its resources.
 
     The cluster directory is used by :command:`ipcontroller`, 
     :command:`ipcontroller` and :command:`ipcontroller` to manage the
     configuration, logging and security of these applications.
 
     This object knows how to find, create and manage these directories. This
     should be used by any code that want's to handle cluster directories.
     """
 
     security_dir_name = Unicode('security')
     log_dir_name = Unicode('log')
     pid_dir_name = Unicode('pid')
     security_dir = Unicode(u'')
     log_dir = Unicode(u'')
     pid_dir = Unicode(u'')
     location = Unicode(u'')
 
     def __init__(self, location):
         super(ClusterDir, self).__init__(None)
         self.location = location
 
     def _location_changed(self, name, old, new):
         if not os.path.isdir(new):
             os.makedirs(new)
         self.security_dir = os.path.join(new, self.security_dir_name)
         self.log_dir = os.path.join(new, self.log_dir_name)
         self.pid_dir = os.path.join(new, self.pid_dir_name)
         self.check_dirs()
 
     def _log_dir_changed(self, name, old, new):
         self.check_log_dir()
 
     def check_log_dir(self):
         if not os.path.isdir(self.log_dir):
             os.mkdir(self.log_dir)
 
     def _security_dir_changed(self, name, old, new):
         self.check_security_dir()
 
     def check_security_dir(self):
         if not os.path.isdir(self.security_dir):
             os.mkdir(self.security_dir, 0700)
         os.chmod(self.security_dir, 0700)
 
     def _pid_dir_changed(self, name, old, new):
         self.check_pid_dir()
 
     def check_pid_dir(self):
         if not os.path.isdir(self.pid_dir):
             os.mkdir(self.pid_dir, 0700)
         os.chmod(self.pid_dir, 0700)
 
     def check_dirs(self):
         self.check_security_dir()
         self.check_log_dir()
         self.check_pid_dir()
 
     def load_config_file(self, filename):
         """Load a config file from the top level of the cluster dir.
 
         Parameters
         ----------
         filename : unicode or str
             The filename only of the config file that must be located in
             the top-level of the cluster directory.
         """
         loader = PyFileConfigLoader(filename, self.location)
         return loader.load_config()
 
     def copy_config_file(self, config_file, path=None, overwrite=False):
         """Copy a default config file into the active cluster directory.
 
         Default configuration files are kept in :mod:`IPython.config.default`.
         This function moves these from that location to the working cluster
         directory.
         """
         if path is None:
             import IPython.config.default
             path = IPython.config.default.__file__.split(os.path.sep)[:-1]
             path = os.path.sep.join(path)
         src = os.path.join(path, config_file)
         dst = os.path.join(self.location, config_file)
         if not os.path.isfile(dst) or overwrite:
             shutil.copy(src, dst)
 
     def copy_all_config_files(self, path=None, overwrite=False):
         """Copy all config files into the active cluster directory."""
         for f in [u'ipcontroller_config.py', u'ipengine_config.py',
                   u'ipcluster_config.py']:
             self.copy_config_file(f, path=path, overwrite=overwrite)
 
     @classmethod
     def create_cluster_dir(csl, cluster_dir):
         """Create a new cluster directory given a full path.
 
         Parameters
         ----------
         cluster_dir : str
             The full path to the cluster directory.  If it does exist, it will
             be used.  If not, it will be created.
         """
         return ClusterDir(cluster_dir)
 
     @classmethod
     def create_cluster_dir_by_profile(cls, path, profile=u'default'):
         """Create a cluster dir by profile name and path.
 
         Parameters
         ----------
         path : str
             The path (directory) to put the cluster directory in.
         profile : str
             The name of the profile.  The name of the cluster directory will
             be "cluster_<profile>".
         """
         if not os.path.isdir(path):
             raise ClusterDirError('Directory not found: %s' % path)
         cluster_dir = os.path.join(path, u'cluster_' + profile)
         return ClusterDir(cluster_dir)
 
     @classmethod
     def find_cluster_dir_by_profile(cls, ipython_dir, profile=u'default'):
         """Find an existing cluster dir by profile name, return its ClusterDir.
 
         This searches through a sequence of paths for a cluster dir.  If it
         is not found, a :class:`ClusterDirError` exception will be raised.
 
         The search path algorithm is:
         1. ``os.getcwd()``
         2. ``ipython_dir``
         3. The directories found in the ":" separated 
            :env:`IPCLUSTER_DIR_PATH` environment variable.
 
         Parameters
         ----------
         ipython_dir : unicode or str
             The IPython directory to use.
         profile : unicode or str
             The name of the profile.  The name of the cluster directory
             will be "cluster_<profile>".
         """
         dirname = u'cluster_' + profile
         cluster_dir_paths = os.environ.get('IPCLUSTER_DIR_PATH','')
         if cluster_dir_paths:
             cluster_dir_paths = cluster_dir_paths.split(':')
         else:
             cluster_dir_paths = []
         paths = [os.getcwd(), ipython_dir] + cluster_dir_paths
         for p in paths:
             cluster_dir = os.path.join(p, dirname)
             if os.path.isdir(cluster_dir):
                 return ClusterDir(cluster_dir)
         else:
             raise ClusterDirError('Cluster directory not found in paths: %s' % dirname)
 
     @classmethod
     def find_cluster_dir(cls, cluster_dir):
         """Find/create a cluster dir and return its ClusterDir.
 
         This will create the cluster directory if it doesn't exist.
 
         Parameters
         ----------
         cluster_dir : unicode or str
             The path of the cluster directory.  This is expanded using
             :func:`IPython.utils.genutils.expand_path`.
         """
         cluster_dir = expand_path(cluster_dir)
         if not os.path.isdir(cluster_dir):
             raise ClusterDirError('Cluster directory not found: %s' % cluster_dir)
         return ClusterDir(cluster_dir)
 
 
 #-----------------------------------------------------------------------------
 # Command line options
 #-----------------------------------------------------------------------------
 
 class ClusterDirConfigLoader(BaseAppConfigLoader):
 
     def _add_cluster_profile(self, parser):
         paa = parser.add_argument
         paa('-p', '--profile',
             dest='Global.profile',type=unicode,
             help=
             """The string name of the profile to be used. This determines the name
             of the cluster dir as: cluster_<profile>. The default profile is named
             'default'.  The cluster directory is resolve this way if the
             --cluster-dir option is not used.""",
             metavar='Global.profile')
 
     def _add_cluster_dir(self, parser):
         paa = parser.add_argument
         paa('--cluster-dir',
             dest='Global.cluster_dir',type=unicode,
             help="""Set the cluster dir. This overrides the logic used by the
             --profile option.""",
             metavar='Global.cluster_dir')
 
     def _add_work_dir(self, parser):
         paa = parser.add_argument
         paa('--work-dir',
             dest='Global.work_dir',type=unicode,
             help='Set the working dir for the process.',
             metavar='Global.work_dir')
 
     def _add_clean_logs(self, parser):
         paa = parser.add_argument
         paa('--clean-logs',
             dest='Global.clean_logs', action='store_true',
             help='Delete old log flies before starting.')
 
     def _add_no_clean_logs(self, parser):
         paa = parser.add_argument
         paa('--no-clean-logs',
             dest='Global.clean_logs', action='store_false',
             help="Don't Delete old log flies before starting.")
 
     def _add_arguments(self):
         super(ClusterDirConfigLoader, self)._add_arguments()
         self._add_cluster_profile(self.parser)
         self._add_cluster_dir(self.parser)
         self._add_work_dir(self.parser)
         self._add_clean_logs(self.parser)
         self._add_no_clean_logs(self.parser)
 
 
 #-----------------------------------------------------------------------------
 # Crash handler for this application
 #-----------------------------------------------------------------------------
 
 
 _message_template = """\
 Oops, $self.app_name crashed. We do our best to make it stable, but...
 
 A crash report was automatically generated with the following information:
   - A verbatim copy of the crash traceback.
   - Data on your current $self.app_name configuration.
 
 It was left in the file named:
 \t'$self.crash_report_fname'
 If you can email this file to the developers, the information in it will help
 them in understanding and correcting the problem.
 
 You can mail it to: $self.contact_name at $self.contact_email
 with the subject '$self.app_name Crash Report'.
 
 If you want to do it now, the following command will work (under Unix):
 mail -s '$self.app_name Crash Report' $self.contact_email < $self.crash_report_fname
 
 To ensure accurate tracking of this issue, please file a report about it at:
 $self.bug_tracker
 """
 
 class ClusterDirCrashHandler(CrashHandler):
     """sys.excepthook for IPython itself, leaves a detailed report on disk."""
 
     message_template = _message_template
 
     def __init__(self, app):
         contact_name = release.authors['Brian'][0]
         contact_email = release.authors['Brian'][1]
         bug_tracker = 'https://bugs.launchpad.net/ipython/+filebug'
         super(ClusterDirCrashHandler,self).__init__(
             app, contact_name, contact_email, bug_tracker
         )
 
 
 #-----------------------------------------------------------------------------
 # Main application
 #-----------------------------------------------------------------------------
 
 class ApplicationWithClusterDir(Application):
     """An application that puts everything into a cluster directory.
 
     Instead of looking for things in the ipython_dir, this type of application
     will use its own private directory called the "cluster directory"
     for things like config files, log files, etc.
 
     The cluster directory is resolved as follows:
 
     * If the ``--cluster-dir`` option is given, it is used.
     * If ``--cluster-dir`` is not given, the application directory is 
       resolve using the profile name as ``cluster_<profile>``. The search 
       path for this directory is then i) cwd if it is found there
       and ii) in ipython_dir otherwise.
 
     The config file for the application is to be put in the cluster
     dir and named the value of the ``config_file_name`` class attribute.
     """
 
     command_line_loader = ClusterDirConfigLoader
     crash_handler_class = ClusterDirCrashHandler
     auto_create_cluster_dir = True
 
     def create_default_config(self):
         super(ApplicationWithClusterDir, self).create_default_config()
         self.default_config.Global.profile = u'default'
         self.default_config.Global.cluster_dir = u''
         self.default_config.Global.work_dir = os.getcwd()
         self.default_config.Global.log_to_file = False
         self.default_config.Global.clean_logs = False
 
     def find_resources(self):
         """This resolves the cluster directory.
 
         This tries to find the cluster directory and if successful, it will
         have done:
         * Sets ``self.cluster_dir_obj`` to the :class:`ClusterDir` object for 
           the application.
         * Sets ``self.cluster_dir`` attribute of the application and config
           objects.
 
         The algorithm used for this is as follows:
         1. Try ``Global.cluster_dir``.
         2. Try using ``Global.profile``.
         3. If both of these fail and ``self.auto_create_cluster_dir`` is
            ``True``, then create the new cluster dir in the IPython directory.
         4. If all fails, then raise :class:`ClusterDirError`.
         """
 
         try:
             cluster_dir = self.command_line_config.Global.cluster_dir
         except AttributeError:
             cluster_dir = self.default_config.Global.cluster_dir
         cluster_dir = expand_path(cluster_dir)
         try:
             self.cluster_dir_obj = ClusterDir.find_cluster_dir(cluster_dir)
         except ClusterDirError:
             pass
         else:
             self.log.info('Using existing cluster dir: %s' % \
                 self.cluster_dir_obj.location
             )
             self.finish_cluster_dir()
             return
 
         try:
             self.profile = self.command_line_config.Global.profile
         except AttributeError:
             self.profile = self.default_config.Global.profile
         try:
             self.cluster_dir_obj = ClusterDir.find_cluster_dir_by_profile(
                 self.ipython_dir, self.profile)
         except ClusterDirError:
             pass
         else:
             self.log.info('Using existing cluster dir: %s' % \
                 self.cluster_dir_obj.location
             )
             self.finish_cluster_dir()
             return
 
         if self.auto_create_cluster_dir:
             self.cluster_dir_obj = ClusterDir.create_cluster_dir_by_profile(
                 self.ipython_dir, self.profile
             )
             self.log.info('Creating new cluster dir: %s' % \
                 self.cluster_dir_obj.location
             )
             self.finish_cluster_dir()
         else:
             raise ClusterDirError('Could not find a valid cluster directory.')
 
     def finish_cluster_dir(self):
         # Set the cluster directory
         self.cluster_dir = self.cluster_dir_obj.location
         
         # These have to be set because they could be different from the one
         # that we just computed.  Because command line has the highest
         # priority, this will always end up in the master_config.
         self.default_config.Global.cluster_dir = self.cluster_dir
         self.command_line_config.Global.cluster_dir = self.cluster_dir
 
     def find_config_file_name(self):
         """Find the config file name for this application."""
         # For this type of Application it should be set as a class attribute.
-        if not hasattr(self, 'config_file_name'):
+        if not hasattr(self, 'default_config_file_name'):
             self.log.critical("No config filename found")
+        else:
+            self.config_file_name = self.default_config_file_name
 
     def find_config_file_paths(self):
         # Set the search path to to the cluster directory. We should NOT
         # include IPython.config.default here as the default config files
         # are ALWAYS automatically moved to the cluster directory.
         conf_dir = os.path.join(get_ipython_package_dir(), 'config', 'default')
         self.config_file_paths = (self.cluster_dir,)
 
     def pre_construct(self):
         # The log and security dirs were set earlier, but here we put them
         # into the config and log them.
         config = self.master_config
         sdir = self.cluster_dir_obj.security_dir
         self.security_dir = config.Global.security_dir = sdir
         ldir = self.cluster_dir_obj.log_dir
         self.log_dir = config.Global.log_dir = ldir
         pdir = self.cluster_dir_obj.pid_dir
         self.pid_dir = config.Global.pid_dir = pdir
         self.log.info("Cluster directory set to: %s" % self.cluster_dir)
         config.Global.work_dir = unicode(expand_path(config.Global.work_dir))
         # Change to the working directory. We do this just before construct
         # is called so all the components there have the right working dir.
         self.to_work_dir()
 
     def to_work_dir(self):
         wd = self.master_config.Global.work_dir
         if unicode(wd) != unicode(os.getcwd()):
             os.chdir(wd)
             self.log.info("Changing to working dir: %s" % wd)
 
     def start_logging(self):
         # Remove old log files
         if self.master_config.Global.clean_logs:
             log_dir = self.master_config.Global.log_dir
             for f in os.listdir(log_dir):
                 if f.startswith(self.name + u'-') and f.endswith('.log'):
                     os.remove(os.path.join(log_dir, f))
         # Start logging to the new log file
         if self.master_config.Global.log_to_file:
             log_filename = self.name + u'-' + str(os.getpid()) + u'.log'
             logfile = os.path.join(self.log_dir, log_filename)
             open_log_file = open(logfile, 'w')
         else:
             open_log_file = sys.stdout
         log.startLogging(open_log_file)
 
     def write_pid_file(self, overwrite=False):
         """Create a .pid file in the pid_dir with my pid.
 
         This must be called after pre_construct, which sets `self.pid_dir`.
         This raises :exc:`PIDFileError` if the pid file exists already.
         """
         pid_file = os.path.join(self.pid_dir, self.name + u'.pid')
         if os.path.isfile(pid_file):
             pid = self.get_pid_from_file()
             if not overwrite:
                 raise PIDFileError(
                     'The pid file [%s] already exists. \nThis could mean that this '
                     'server is already running with [pid=%s].' % (pid_file, pid)
                 )
         with open(pid_file, 'w') as f:
             self.log.info("Creating pid file: %s" % pid_file)
             f.write(repr(os.getpid())+'\n')
 
     def remove_pid_file(self):
         """Remove the pid file.
 
         This should be called at shutdown by registering a callback with
         :func:`reactor.addSystemEventTrigger`. This needs to return
         ``None``.
         """
         pid_file = os.path.join(self.pid_dir, self.name + u'.pid')
         if os.path.isfile(pid_file):
             try:
                 self.log.info("Removing pid file: %s" % pid_file)
                 os.remove(pid_file)
             except:
                 self.log.warn("Error removing the pid file: %s" % pid_file)
 
     def get_pid_from_file(self):
         """Get the pid from the pid file.
 
         If the  pid file doesn't exist a :exc:`PIDFileError` is raised.
         """
         pid_file = os.path.join(self.pid_dir, self.name + u'.pid')
         if os.path.isfile(pid_file):
             with open(pid_file, 'r') as f:
                 pid = int(f.read().strip())
                 return pid
         else:
             raise PIDFileError('pid file not found: %s' % pid_file)
 

IPython/kernel/fcutil.py

@@ -1,273 +1,277 b''
 #!/usr/bin/env python
 # encoding: utf-8
 """
 Foolscap related utilities.
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 from __future__ import with_statement
 
 import os
 import tempfile
 
 from twisted.internet import reactor, defer
 from twisted.python import log
 
 from foolscap import Tub, UnauthenticatedTub
 
 from IPython.config.loader import Config
 
 from IPython.kernel.configobjfactory import AdaptedConfiguredObjectFactory
 
 from IPython.kernel.error import SecurityError
 
 from IPython.utils.traitlets import Int, Str, Bool, Instance
 from IPython.utils.importstring import import_item
 
 #-----------------------------------------------------------------------------
 # Code
 #-----------------------------------------------------------------------------
 
 
 # We do this so if a user doesn't have OpenSSL installed, it will try to use
 # an UnauthenticatedTub.  But, they will still run into problems if they
 # try to use encrypted furls.
 try:
     import OpenSSL
 except:
     Tub = UnauthenticatedTub
     have_crypto = False
 else:
     have_crypto = True
 
 
 class FURLError(Exception):
     pass
 
 
 def check_furl_file_security(furl_file, secure):
     """Remove the old furl_file if changing security modes."""
     if os.path.isfile(furl_file):
         f = open(furl_file, 'r')
         oldfurl = f.read().strip()
         f.close()
         if (oldfurl.startswith('pb://') and not secure) or (oldfurl.startswith('pbu://') and secure):
             os.remove(furl_file)
 
 
 def is_secure(furl):
     """Is the given FURL secure or not."""
     if is_valid(furl):
         if furl.startswith("pb://"):
             return True
         elif furl.startswith("pbu://"):
             return False
     else:
         raise FURLError("invalid FURL: %s" % furl)
 
 
 def is_valid(furl):
     """Is the str a valid FURL or not."""
     if isinstance(furl, str):
         if furl.startswith("pb://") or furl.startswith("pbu://"):
             return True
     else:
         return False
 
 
 def find_furl(furl_or_file):
     """Find, validate and return a FURL in a string or file."""
     if isinstance(furl_or_file, str):
         if is_valid(furl_or_file):
             return furl_or_file
     if os.path.isfile(furl_or_file):
         with open(furl_or_file, 'r') as f:
             furl = f.read().strip()
         if is_valid(furl):
             return furl
     raise FURLError("Not a valid FURL or FURL file: %s" % furl_or_file)
 
 
 def is_valid_furl_or_file(furl_or_file):
     """Validate a FURL or a FURL file.
 
     If ``furl_or_file`` looks like a file, we simply make sure its directory
     exists and that it has a ``.furl`` file extension.  We don't try to see
     if the FURL file exists or to read its contents. This is useful for
     cases where auto re-connection is being used.
     """
     if isinstance(furl_or_file, str):
         if is_valid(furl_or_file):
             return True
     if isinstance(furl_or_file, (str, unicode)):
         path, furl_filename = os.path.split(furl_or_file)
         if os.path.isdir(path) and furl_filename.endswith('.furl'):
             return True
     return False
 
 
 def validate_furl_or_file(furl_or_file):
     if not is_valid_furl_or_file(furl_or_file):
         raise FURLError('Not a valid FURL or FURL file: %r' % furl_or_file)
 
 
 def get_temp_furlfile(filename):
     """Return a temporary FURL file."""
     return tempfile.mktemp(dir=os.path.dirname(filename),
                            prefix=os.path.basename(filename))
 
 
 def make_tub(ip, port, secure, cert_file):
     """Create a listening tub given an ip, port, and cert_file location.
     
     Parameters
     ----------
     ip : str
         The ip address or hostname that the tub should listen on.  
         Empty means all interfaces.
     port : int
         The port that the tub should listen on.  A value of 0 means
         pick a random port
     secure: bool
         Will the connection be secure (in the Foolscap sense).
     cert_file: str
         A filename of a file to be used for theSSL certificate.
 
     Returns
     -------
     A tub, listener tuple.
     """
     if secure:
         if have_crypto:
             tub = Tub(certFile=cert_file)
         else:
             raise SecurityError("OpenSSL/pyOpenSSL is not available, so we "
                                 "can't run in secure mode.  Try running without "
                                 "security using 'ipcontroller -xy'.")
     else:
         tub = UnauthenticatedTub()
     
     # Set the strport based on the ip and port and start listening
     if ip == '':
         strport = "tcp:%i" % port
     else:
         strport = "tcp:%i:interface=%s" % (port, ip)
     log.msg("Starting listener with [secure=%r] on: %s" % (secure, strport))
     listener = tub.listenOn(strport)
     
     return tub, listener
 
 
 class FCServiceFactory(AdaptedConfiguredObjectFactory):
     """This class creates a tub with various services running in it.
 
     The basic idea is that :meth:`create` returns a running :class:`Tub`
     instance that has a number of Foolscap references registered in it.
     This class is a subclass of :class:`IPython.core.component.Component`
     so the IPython configuration and component system are used.
 
     Attributes
     ----------
     interfaces : Config
         A Config instance whose values are sub-Config objects having two
         keys: furl_file and interface_chain.
 
     The other attributes are the standard ones for Foolscap.
     """
 
     ip = Str('', config=True)
     port = Int(0, config=True)
     secure = Bool(True, config=True)
     cert_file = Str('', config=True)
     location = Str('', config=True)
     reuse_furls = Bool(False, config=True)
     interfaces = Instance(klass=Config, kw={}, allow_none=False, config=True)
 
     def __init__(self, config, adaptee):
         super(FCServiceFactory, self).__init__(config, adaptee)
         self._check_reuse_furls()
 
     def _ip_changed(self, name, old, new):
         if new == 'localhost' or new == '127.0.0.1':
             self.location = '127.0.0.1'
 
     def _check_reuse_furls(self):
         furl_files = [i.furl_file for i in self.interfaces.values()]
         for ff in furl_files:
             fullfile = self._get_security_file(ff)
             if self.reuse_furls:
                 if self.port==0:
                     raise FURLError("You are trying to reuse the FURL file "
                         "for this connection, but the port for this connection "
                         "is set to 0 (autoselect). To reuse the FURL file "
                         "you need to specify specific port to listen on."
                     )
                 else:
                     log.msg("Reusing FURL file: %s" % fullfile)
             else:
                 if os.path.isfile(fullfile):
                     log.msg("Removing old FURL file: %s" % fullfile)
                     os.remove(fullfile)
 
     def _get_security_file(self, filename):
         return os.path.join(self.config.Global.security_dir, filename)
 
     def create(self):
         """Create and return the Foolscap tub with everything running."""
 
         self.tub, self.listener = make_tub(
             self.ip, self.port, self.secure, 
             self._get_security_file(self.cert_file)
         )
         # log.msg("Interfaces to register [%r]: %r" % \
         #     (self.__class__, self.interfaces))
         if not self.secure:
             log.msg("WARNING: running with no security: %s" % \
                 self.__class__.__name__)
         reactor.callWhenRunning(self.set_location_and_register)
         return self.tub
 
     def set_location_and_register(self):
         """Set the location for the tub and return a deferred."""
 
         if self.location == '':
             d = self.tub.setLocationAutomatically()
         else:
             d = defer.maybeDeferred(self.tub.setLocation,
                 "%s:%i" % (self.location, self.listener.getPortnum()))
         self.adapt_to_interfaces(d)
 
     def adapt_to_interfaces(self, d):
         """Run through the interfaces, adapt and register."""
 
         for ifname, ifconfig in self.interfaces.iteritems():
             ff = self._get_security_file(ifconfig.furl_file)
             log.msg("Adapting [%s] to interface: %s" % \
                 (self.adaptee.__class__.__name__, ifname))
             log.msg("Saving FURL for interface [%s] to file: %s" % (ifname, ff))
             check_furl_file_security(ff, self.secure)
             adaptee = self.adaptee
             for i in ifconfig.interface_chain:
                 adaptee = import_item(i)(adaptee)
             d.addCallback(self.register, adaptee, furl_file=ff)
 
     def register(self, empty, ref, furl_file):
         """Register the reference with the FURL file.
 
         The FURL file is created and then moved to make sure that when the
-        file appears, the buffer has been flushed and the file closed.
+        file appears, the buffer has been flushed and the file closed. This
+        is not done if we are re-using FURLS however.
         """
-        temp_furl_file = get_temp_furlfile(furl_file)
-        self.tub.registerReference(ref, furlFile=temp_furl_file)
-        os.rename(temp_furl_file, furl_file)
+        if self.reuse_furls:
+            self.tub.registerReference(ref, furlFile=furl_file)
+        else:
+            temp_furl_file = get_temp_furlfile(furl_file)
+            self.tub.registerReference(ref, furlFile=temp_furl_file)
+            os.rename(temp_furl_file, furl_file)
 

IPython/kernel/ipclusterapp.py

@@ -1,499 +1,499 b''
 #!/usr/bin/env python
 # encoding: utf-8
 """
 The ipcluster application.
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import logging
 import os
 import signal
 
 if os.name=='posix':
     from twisted.scripts._twistd_unix import daemonize
 
 from twisted.internet import reactor, defer
 from twisted.python import log, failure
 
 
 from IPython.external.argparse import ArgumentParser, SUPPRESS
 from IPython.utils.importstring import import_item
 from IPython.kernel.clusterdir import (
     ApplicationWithClusterDir, ClusterDirConfigLoader,
     ClusterDirError, PIDFileError
 )
 
 
 #-----------------------------------------------------------------------------
 # Module level variables
 #-----------------------------------------------------------------------------
 
 
 default_config_file_name = u'ipcluster_config.py'
 
 
 _description = """\
 Start an IPython cluster for parallel computing.\n\n
 
 An IPython cluster consists of 1 controller and 1 or more engines.
 This command automates the startup of these processes using a wide
 range of startup methods (SSH, local processes, PBS, mpiexec,
 Windows HPC Server 2008). To start a cluster with 4 engines on your
 local host simply do 'ipcluster start -n 4'. For more complex usage 
 you will typically do 'ipcluster create -p mycluster', then edit
 configuration files, followed by 'ipcluster start -p mycluster -n 4'.
 """
 
 
 # Exit codes for ipcluster
 
 # This will be the exit code if the ipcluster appears to be running because
 # a .pid file exists
 ALREADY_STARTED = 10
 
 
 # This will be the exit code if ipcluster stop is run, but there is not .pid
 # file to be found.
 ALREADY_STOPPED = 11
 
 
 #-----------------------------------------------------------------------------
 # Command line options
 #-----------------------------------------------------------------------------
 
 
 class IPClusterAppConfigLoader(ClusterDirConfigLoader):
 
     def _add_arguments(self):
         # Don't call ClusterDirConfigLoader._add_arguments as we don't want
         # its defaults on self.parser. Instead, we will put those on
         # default options on our subparsers.
         
         # This has all the common options that all subcommands use
         parent_parser1 = ArgumentParser(
             add_help=False,
             argument_default=SUPPRESS
         )
         self._add_ipython_dir(parent_parser1)
         self._add_log_level(parent_parser1)
 
         # This has all the common options that other subcommands use
         parent_parser2 = ArgumentParser(
             add_help=False,
             argument_default=SUPPRESS
         )
         self._add_cluster_profile(parent_parser2)
         self._add_cluster_dir(parent_parser2)
         self._add_work_dir(parent_parser2)
         paa = parent_parser2.add_argument
         paa('--log-to-file',
             action='store_true', dest='Global.log_to_file', 
             help='Log to a file in the log directory (default is stdout)')
 
         # Create the object used to create the subparsers.
         subparsers = self.parser.add_subparsers(
             dest='Global.subcommand',
             title='ipcluster subcommands',
             description=
             """ipcluster has a variety of subcommands. The general way of 
             running ipcluster is 'ipcluster <cmd> [options]'. To get help
             on a particular subcommand do 'ipcluster <cmd> -h'."""
             # help="For more help, type 'ipcluster <cmd> -h'",
         )
 
         # The "list" subcommand parser
         parser_list = subparsers.add_parser(
             'list',
             parents=[parent_parser1],
             argument_default=SUPPRESS,
             help="List all clusters in cwd and ipython_dir.",
             description=
             """List all available clusters, by cluster directory, that can
             be found in the current working directly or in the ipython
             directory. Cluster directories are named using the convention
             'cluster_<profile>'."""
         )
 
         # The "create" subcommand parser
         parser_create = subparsers.add_parser(
             'create',
             parents=[parent_parser1, parent_parser2],
             argument_default=SUPPRESS,
             help="Create a new cluster directory.",
             description=
             """Create an ipython cluster directory by its profile name or 
             cluster directory path. Cluster directories contain 
             configuration, log and security related files and are named 
             using the convention 'cluster_<profile>'. By default they are
             located in your ipython directory. Once created, you will
             probably need to edit the configuration files in the cluster
             directory to configure your cluster. Most users will create a
             cluster directory by profile name, 
             'ipcluster create -p mycluster', which will put the directory
             in '<ipython_dir>/cluster_mycluster'. 
             """
         )
         paa = parser_create.add_argument
         paa('--reset-config',
             dest='Global.reset_config', action='store_true',
             help=
             """Recopy the default config files to the cluster directory.
             You will loose any modifications you have made to these files.""")
 
         # The "start" subcommand parser
         parser_start = subparsers.add_parser(
             'start',
             parents=[parent_parser1, parent_parser2],
             argument_default=SUPPRESS,
             help="Start a cluster.",
             description=
             """Start an ipython cluster by its profile name or cluster 
             directory. Cluster directories contain configuration, log and
             security related files and are named using the convention
             'cluster_<profile>' and should be creating using the 'start'
             subcommand of 'ipcluster'. If your cluster directory is in 
             the cwd or the ipython directory, you can simply refer to it
             using its profile name, 'ipcluster start -n 4 -p <profile>`,
             otherwise use the '--cluster-dir' option.
             """
         )
         paa = parser_start.add_argument
         paa('-n', '--number',
             type=int, dest='Global.n',
             help='The number of engines to start.',
             metavar='Global.n')
         paa('--clean-logs',
             dest='Global.clean_logs', action='store_true',
             help='Delete old log flies before starting.')
         paa('--no-clean-logs',
             dest='Global.clean_logs', action='store_false',
             help="Don't delete old log flies before starting.")
         paa('--daemon',
             dest='Global.daemonize', action='store_true',
             help='Daemonize the ipcluster program. This implies --log-to-file')
         paa('--no-daemon',
             dest='Global.daemonize', action='store_false',
             help="Dont't daemonize the ipcluster program.")
 
         # The "stop" subcommand parser
         parser_stop = subparsers.add_parser(
             'stop',
             parents=[parent_parser1, parent_parser2],
             argument_default=SUPPRESS,
             help="Stop a running cluster.",
             description=
             """Stop a running ipython cluster by its profile name or cluster 
             directory. Cluster directories are named using the convention
             'cluster_<profile>'. If your cluster directory is in 
             the cwd or the ipython directory, you can simply refer to it
             using its profile name, 'ipcluster stop -p <profile>`, otherwise
             use the '--cluster-dir' option.
             """
         )
         paa = parser_stop.add_argument
         paa('--signal',
             dest='Global.signal', type=int,
             help="The signal number to use in stopping the cluster (default=2).",
             metavar="Global.signal")
 
 
 #-----------------------------------------------------------------------------
 # Main application
 #-----------------------------------------------------------------------------
 
 
 class IPClusterApp(ApplicationWithClusterDir):
 
     name = u'ipcluster'
     description = _description
     usage = None
     command_line_loader = IPClusterAppConfigLoader
-    config_file_name = default_config_file_name
+    default_config_file_name = default_config_file_name
     default_log_level = logging.INFO
     auto_create_cluster_dir = False
 
     def create_default_config(self):
         super(IPClusterApp, self).create_default_config()
         self.default_config.Global.controller_launcher = \
             'IPython.kernel.launcher.LocalControllerLauncher'
         self.default_config.Global.engine_launcher = \
             'IPython.kernel.launcher.LocalEngineSetLauncher'
         self.default_config.Global.n = 2
         self.default_config.Global.reset_config = False
         self.default_config.Global.clean_logs = True
         self.default_config.Global.signal = 2
         self.default_config.Global.daemonize = False
 
     def find_resources(self):
         subcommand = self.command_line_config.Global.subcommand
         if subcommand=='list':
             self.list_cluster_dirs()
             # Exit immediately because there is nothing left to do.
             self.exit()
         elif subcommand=='create':
             self.auto_create_cluster_dir = True
             super(IPClusterApp, self).find_resources()
         elif subcommand=='start' or subcommand=='stop':
             self.auto_create_cluster_dir = True
             try:
                 super(IPClusterApp, self).find_resources()
             except ClusterDirError:
                 raise ClusterDirError(
                     "Could not find a cluster directory. A cluster dir must "
                     "be created before running 'ipcluster start'.  Do "
                     "'ipcluster create -h' or 'ipcluster list -h' for more "
                     "information about creating and listing cluster dirs."
                 )
 
     def list_cluster_dirs(self):
         # Find the search paths
         cluster_dir_paths = os.environ.get('IPCLUSTER_DIR_PATH','')
         if cluster_dir_paths:
             cluster_dir_paths = cluster_dir_paths.split(':')
         else:
             cluster_dir_paths = []
         try:
             ipython_dir = self.command_line_config.Global.ipython_dir
         except AttributeError:
             ipython_dir = self.default_config.Global.ipython_dir
         paths = [os.getcwd(), ipython_dir] + \
             cluster_dir_paths
         paths = list(set(paths))
 
         self.log.info('Searching for cluster dirs in paths: %r' % paths)
         for path in paths:
             files = os.listdir(path)
             for f in files:
                 full_path = os.path.join(path, f)
                 if os.path.isdir(full_path) and f.startswith('cluster_'):
                     profile = full_path.split('_')[-1]
                     start_cmd = 'ipcluster start -p %s -n 4' % profile
                     print start_cmd + " ==> " + full_path
 
     def pre_construct(self):
         # IPClusterApp.pre_construct() is where we cd to the working directory.
         super(IPClusterApp, self).pre_construct()
         config = self.master_config
         try:
             daemon = config.Global.daemonize
             if daemon:
                 config.Global.log_to_file = True
         except AttributeError:
             pass
 
     def construct(self):
         config = self.master_config
         subcmd = config.Global.subcommand
         reset = config.Global.reset_config
         if subcmd == 'list':
             return
         if subcmd == 'create':
             self.log.info('Copying default config files to cluster directory '
             '[overwrite=%r]' % (reset,))
             self.cluster_dir_obj.copy_all_config_files(overwrite=reset)
         if subcmd =='start':
             self.cluster_dir_obj.copy_all_config_files(overwrite=False)
             self.start_logging()
             reactor.callWhenRunning(self.start_launchers)
 
     def start_launchers(self):
         config = self.master_config
 
         # Create the launchers. In both bases, we set the work_dir of
         # the launcher to the cluster_dir. This is where the launcher's
         # subprocesses will be launched. It is not where the controller
         # and engine will be launched.
         el_class = import_item(config.Global.engine_launcher)
         self.engine_launcher = el_class(
             work_dir=self.cluster_dir, config=config
         )
         cl_class = import_item(config.Global.controller_launcher)
         self.controller_launcher = cl_class(
             work_dir=self.cluster_dir, config=config
         )
 
         # Setup signals
         signal.signal(signal.SIGINT, self.sigint_handler)
 
         # Setup the observing of stopping. If the controller dies, shut
         # everything down as that will be completely fatal for the engines.
         d1 = self.controller_launcher.observe_stop()
         d1.addCallback(self.stop_launchers)
         # But, we don't monitor the stopping of engines. An engine dying
         # is just fine and in principle a user could start a new engine.
         # Also, if we did monitor engine stopping, it is difficult to
         # know what to do when only some engines die. Currently, the
         # observing of engine stopping is inconsistent. Some launchers
         # might trigger on a single engine stopping, other wait until
         # all stop.  TODO: think more about how to handle this.
         
         # Start the controller and engines
         self._stopping = False  # Make sure stop_launchers is not called 2x.
         d = self.start_controller()
         d.addCallback(self.start_engines)
         d.addCallback(self.startup_message)
         # If the controller or engines fail to start, stop everything
         d.addErrback(self.stop_launchers)
         return d
 
     def startup_message(self, r=None):
         log.msg("IPython cluster: started")
         return r
         
     def start_controller(self, r=None):
         # log.msg("In start_controller")
         config = self.master_config
         d = self.controller_launcher.start(
             cluster_dir=config.Global.cluster_dir
         )
         return d
             
     def start_engines(self, r=None):
         # log.msg("In start_engines")
         config = self.master_config
         d = self.engine_launcher.start(
             config.Global.n,
             cluster_dir=config.Global.cluster_dir
         )
         return d
 
     def stop_controller(self, r=None):
         # log.msg("In stop_controller")
         if self.controller_launcher.running:
             d = self.controller_launcher.stop()
             d.addErrback(self.log_err)
             return d
         else:
             return defer.succeed(None)
 
     def stop_engines(self, r=None):
         # log.msg("In stop_engines")
         if self.engine_launcher.running:
             d = self.engine_launcher.stop()
             d.addErrback(self.log_err)
             return d
         else:
             return defer.succeed(None)
 
     def log_err(self, f):
         log.msg(f.getTraceback())
         return None
         
     def stop_launchers(self, r=None):
         if not self._stopping:
             self._stopping = True
             if isinstance(r, failure.Failure):
                 log.msg('Unexpected error in ipcluster:')
                 log.msg(r.getTraceback())
             log.msg("IPython cluster: stopping")
             # These return deferreds. We are not doing anything with them
             # but we are holding refs to them as a reminder that they 
             # do return deferreds.
             d1 = self.stop_engines()
             d2 = self.stop_controller()
             # Wait a few seconds to let things shut down.
             reactor.callLater(4.0, reactor.stop)
 
     def sigint_handler(self, signum, frame):
         self.stop_launchers()
         
     def start_logging(self):
         # Remove old log files of the controller and engine
         if self.master_config.Global.clean_logs:
             log_dir = self.master_config.Global.log_dir
             for f in os.listdir(log_dir):
                 if f.startswith('ipengine' + '-'):
                     if f.endswith('.log') or f.endswith('.out') or f.endswith('.err'):
                         os.remove(os.path.join(log_dir, f))
                 if f.startswith('ipcontroller' + '-'):
                     if f.endswith('.log') or f.endswith('.out') or f.endswith('.err'):
                         os.remove(os.path.join(log_dir, f))
         # This will remote old log files for ipcluster itself
         super(IPClusterApp, self).start_logging()
 
     def start_app(self):
         """Start the application, depending on what subcommand is used."""
         subcmd = self.master_config.Global.subcommand
         if subcmd=='create' or subcmd=='list':
             return
         elif subcmd=='start':
             self.start_app_start()
         elif subcmd=='stop':
             self.start_app_stop()
 
     def start_app_start(self):
         """Start the app for the start subcommand."""
         config = self.master_config
         # First see if the cluster is already running
         try:
             pid = self.get_pid_from_file()
         except PIDFileError:
             pass
         else:
             self.log.critical(
                 'Cluster is already running with [pid=%s]. '
                 'use "ipcluster stop" to stop the cluster.' % pid
             )
             # Here I exit with a unusual exit status that other processes
             # can watch for to learn how I existed.
             self.exit(ALREADY_STARTED)
 
         # Now log and daemonize
         self.log.info(
             'Starting ipcluster with [daemon=%r]' % config.Global.daemonize
         )
         # TODO: Get daemonize working on Windows or as a Windows Server.
         if config.Global.daemonize:
             if os.name=='posix':
                 daemonize()
 
         # Now write the new pid file AFTER our new forked pid is active.
         self.write_pid_file()
         reactor.addSystemEventTrigger('during','shutdown', self.remove_pid_file)
         reactor.run()
 
     def start_app_stop(self):
         """Start the app for the stop subcommand."""
         config = self.master_config
         try:
             pid = self.get_pid_from_file()
         except PIDFileError:
             self.log.critical(
                 'Problem reading pid file, cluster is probably not running.'
             )
             # Here I exit with a unusual exit status that other processes
             # can watch for to learn how I existed.
             self.exit(ALREADY_STOPPED)
         else:
             if os.name=='posix':
                 sig = config.Global.signal
                 self.log.info(
                     "Stopping cluster [pid=%r] with [signal=%r]" % (pid, sig)
                 )
                 os.kill(pid, sig)
             elif os.name=='nt':
                 # As of right now, we don't support daemonize on Windows, so
                 # stop will not do anything. Minimally, it should clean up the
                 # old .pid files.
                 self.remove_pid_file()
 
 
 def launch_new_instance():
     """Create and run the IPython cluster."""
     app = IPClusterApp()
     app.start()
 
 
 if __name__ == '__main__':
     launch_new_instance()
 

IPython/kernel/ipcontrollerapp.py

@@ -1,260 +1,271 b''
 #!/usr/bin/env python
 # encoding: utf-8
 """
 The IPython controller application.
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 from __future__ import with_statement
 
 import copy
 import sys
 
 from twisted.application import service
 from twisted.internet import reactor
 from twisted.python import log
 
 from IPython.config.loader import Config
 from IPython.kernel import controllerservice
 from IPython.kernel.clusterdir import (
     ApplicationWithClusterDir,
     ClusterDirConfigLoader
 )
-from IPython.kernel.fcutil import FCServiceFactory
+from IPython.kernel.fcutil import FCServiceFactory, FURLError
 from IPython.utils.traitlets import Instance, Unicode
 
 
 #-----------------------------------------------------------------------------
 # Module level variables
 #-----------------------------------------------------------------------------
 
 
 #: The default config file name for this application
 default_config_file_name = u'ipcontroller_config.py'
 
 
 _description = """Start the IPython controller for parallel computing.
 
 The IPython controller provides a gateway between the IPython engines and
 clients. The controller needs to be started before the engines and can be
 configured using command line options or using a cluster directory. Cluster
 directories contain config, log and security files and are usually located in
 your .ipython directory and named as "cluster_<profile>". See the --profile
 and --cluster-dir options for details.
 """
 
 #-----------------------------------------------------------------------------
 # Default interfaces
 #-----------------------------------------------------------------------------
 
 # The default client interfaces for FCClientServiceFactory.interfaces
 default_client_interfaces = Config()
 default_client_interfaces.Task.interface_chain = [
     'IPython.kernel.task.ITaskController',
     'IPython.kernel.taskfc.IFCTaskController'
 ]
 
 default_client_interfaces.Task.furl_file = 'ipcontroller-tc.furl'
 
 default_client_interfaces.MultiEngine.interface_chain = [
     'IPython.kernel.multiengine.IMultiEngine',
     'IPython.kernel.multienginefc.IFCSynchronousMultiEngine'
 ]
 
 default_client_interfaces.MultiEngine.furl_file = u'ipcontroller-mec.furl'
 
 # Make this a dict we can pass to Config.__init__ for the default
 default_client_interfaces = dict(copy.deepcopy(default_client_interfaces.items()))
 
 
 
 # The default engine interfaces for FCEngineServiceFactory.interfaces
 default_engine_interfaces = Config()
 default_engine_interfaces.Default.interface_chain = [
     'IPython.kernel.enginefc.IFCControllerBase'
 ]
 
 default_engine_interfaces.Default.furl_file = u'ipcontroller-engine.furl'
 
 # Make this a dict we can pass to Config.__init__ for the default
 default_engine_interfaces = dict(copy.deepcopy(default_engine_interfaces.items()))
 
 
 #-----------------------------------------------------------------------------
 # Service factories
 #-----------------------------------------------------------------------------
 
 
 class FCClientServiceFactory(FCServiceFactory):
     """A Foolscap implementation of the client services."""
 
     cert_file = Unicode(u'ipcontroller-client.pem', config=True)
     interfaces = Instance(klass=Config, kw=default_client_interfaces,
                           allow_none=False, config=True)
 
 
 class FCEngineServiceFactory(FCServiceFactory):
     """A Foolscap implementation of the engine services."""
 
     cert_file = Unicode(u'ipcontroller-engine.pem', config=True)
     interfaces = Instance(klass=dict, kw=default_engine_interfaces,
                           allow_none=False, config=True)
 
 
 #-----------------------------------------------------------------------------
 # Command line options
 #-----------------------------------------------------------------------------
 
 
 class IPControllerAppConfigLoader(ClusterDirConfigLoader):
 
     def _add_arguments(self):
         super(IPControllerAppConfigLoader, self)._add_arguments()
         paa = self.parser.add_argument
         # Client config
         paa('--client-ip',
             type=str, dest='FCClientServiceFactory.ip', 
             help='The IP address or hostname the controller will listen on for '
             'client connections.',
             metavar='FCClientServiceFactory.ip')
         paa('--client-port',
             type=int, dest='FCClientServiceFactory.port', 
             help='The port the controller will listen on for client connections. '
             'The default is to use 0, which will autoselect an open port.',
             metavar='FCClientServiceFactory.port')
         paa('--client-location',), dict(
             type=str, dest='FCClientServiceFactory.location',
             help='The hostname or IP that clients should connect to. This does '
             'not control which interface the controller listens on. Instead, this '
             'determines the hostname/IP that is listed in the FURL, which is how '
             'clients know where to connect. Useful if the controller is listening '
             'on multiple interfaces.',
             metavar='FCClientServiceFactory.location')
         # Engine config
         paa('--engine-ip',
             type=str, dest='FCEngineServiceFactory.ip', 
             help='The IP address or hostname the controller will listen on for '
             'engine connections.',
             metavar='FCEngineServiceFactory.ip')
         paa('--engine-port',
             type=int, dest='FCEngineServiceFactory.port',
             help='The port the controller will listen on for engine connections. '
             'The default is to use 0, which will autoselect an open port.',
             metavar='FCEngineServiceFactory.port')
         paa('--engine-location',
             type=str, dest='FCEngineServiceFactory.location',
             help='The hostname or IP that engines should connect to. This does '
             'not control which interface the controller listens on. Instead, this '
             'determines the hostname/IP that is listed in the FURL, which is how '
             'engines know where to connect. Useful if the controller is listening '
             'on multiple interfaces.',
             metavar='FCEngineServiceFactory.location')
         # Global config
         paa('--log-to-file',
             action='store_true', dest='Global.log_to_file',
             help='Log to a file in the log directory (default is stdout)')
         paa('-r','--reuse-furls', 
             action='store_true', dest='Global.reuse_furls',
             help='Try to reuse all FURL files. If this is not set all FURL files '
             'are deleted before the controller starts. This must be set if '
             'specific ports are specified by --engine-port or --client-port.')
         paa('--no-secure',
             action='store_false', dest='Global.secure',
             help='Turn off SSL encryption for all connections.')
         paa('--secure',
             action='store_true', dest='Global.secure',
             help='Turn off SSL encryption for all connections.')
 
 
 #-----------------------------------------------------------------------------
 # The main application
 #-----------------------------------------------------------------------------
 
 
 class IPControllerApp(ApplicationWithClusterDir):
 
     name = u'ipcontroller'
     description = _description
     command_line_loader = IPControllerAppConfigLoader
-    config_file_name = default_config_file_name
+    default_config_file_name = default_config_file_name
     auto_create_cluster_dir = True
 
     def create_default_config(self):
         super(IPControllerApp, self).create_default_config()
-        self.default_config.Global.reuse_furls = False
-        self.default_config.Global.secure = True
+        # Don't set defaults for Global.secure or Global.reuse_furls
+        # as those are set in a component.
         self.default_config.Global.import_statements = []
         self.default_config.Global.clean_logs = True
 
-    def post_load_command_line_config(self):
-        # Now setup reuse_furls
-        c = self.command_line_config
+    def pre_construct(self):
+        super(IPControllerApp, self).pre_construct()
+        c = self.master_config
+        # The defaults for these are set in FCClientServiceFactory and
+        # FCEngineServiceFactory, so we only set them here if the global
+        # options have be set to override the class level defaults.
         if hasattr(c.Global, 'reuse_furls'):
             c.FCClientServiceFactory.reuse_furls = c.Global.reuse_furls
             c.FCEngineServiceFactory.reuse_furls = c.Global.reuse_furls
             del c.Global.reuse_furls
         if hasattr(c.Global, 'secure'):
             c.FCClientServiceFactory.secure = c.Global.secure
             c.FCEngineServiceFactory.secure = c.Global.secure
             del c.Global.secure
 
     def construct(self):
         # This is the working dir by now.
         sys.path.insert(0, '')
 
         self.start_logging()
         self.import_statements()
 
         # Create the service hierarchy
         self.main_service = service.MultiService()
         # The controller service
         controller_service = controllerservice.ControllerService()
         controller_service.setServiceParent(self.main_service)
         # The client tub and all its refereceables
-        csfactory = FCClientServiceFactory(self.master_config, controller_service)
+        try:
+            csfactory = FCClientServiceFactory(self.master_config, controller_service)
+        except FURLError, e:
+            log.err(e)
+            self.exit(0)
         client_service = csfactory.create()
         client_service.setServiceParent(self.main_service)
         # The engine tub
-        esfactory = FCEngineServiceFactory(self.master_config, controller_service)
+        try:
+            esfactory = FCEngineServiceFactory(self.master_config, controller_service)
+        except FURLError, e:
+            log.err(e)
+            self.exit(0)
         engine_service = esfactory.create()
         engine_service.setServiceParent(self.main_service)
 
     def import_statements(self):
         statements = self.master_config.Global.import_statements
         for s in statements:
             try:
                 log.msg("Executing statement: '%s'" % s)
                 exec s in globals(), locals()
             except:
                 log.msg("Error running statement: %s" % s)
 
     def start_app(self):
         # Start the controller service.
         self.main_service.startService()
         # Write the .pid file overwriting old ones. This allow multiple
         # controllers to clober each other. But Windows is not cleaning
         # these up properly.
         self.write_pid_file(overwrite=True)
         # Add a trigger to delete the .pid file upon shutting down.
         reactor.addSystemEventTrigger('during','shutdown', self.remove_pid_file)
         reactor.run()
 
 
 def launch_new_instance():
     """Create and run the IPython controller"""
     app = IPControllerApp()
     app.start()
 
 
 if __name__ == '__main__':
     launch_new_instance()

IPython/kernel/ipengineapp.py

@@ -1,242 +1,242 b''
 #!/usr/bin/env python
 # encoding: utf-8
 """
 The IPython controller application
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import os
 import sys
 
 from twisted.application import service
 from twisted.internet import reactor
 from twisted.python import log
 
 from IPython.kernel.clusterdir import (
     ApplicationWithClusterDir,
     ClusterDirConfigLoader
 )
 from IPython.kernel.engineconnector import EngineConnector
 from IPython.kernel.engineservice import EngineService
 from IPython.kernel.fcutil import Tub
 from IPython.utils.importstring import import_item
 
 #-----------------------------------------------------------------------------
 # Module level variables
 #-----------------------------------------------------------------------------
 
 #: The default config file name for this application
 default_config_file_name = u'ipengine_config.py'
 
 
 mpi4py_init = """from mpi4py import MPI as mpi
 mpi.size = mpi.COMM_WORLD.Get_size()
 mpi.rank = mpi.COMM_WORLD.Get_rank()
 """
 
 
 pytrilinos_init = """from PyTrilinos import Epetra
 class SimpleStruct:
 pass
 mpi = SimpleStruct()
 mpi.rank = 0
 mpi.size = 0
 """
 
 
 _description = """Start an IPython engine for parallel computing.\n\n
 
 IPython engines run in parallel and perform computations on behalf of a client
 and controller. A controller needs to be started before the engines. The
 engine can be configured using command line options or using a cluster
 directory. Cluster directories contain config, log and security files and are
 usually located in your .ipython directory and named as "cluster_<profile>".
 See the --profile and --cluster-dir options for details.
 """
 
 #-----------------------------------------------------------------------------
 # Command line options
 #-----------------------------------------------------------------------------
 
 
 class IPEngineAppConfigLoader(ClusterDirConfigLoader):
 
     def _add_arguments(self):
         super(IPEngineAppConfigLoader, self)._add_arguments()
         paa = self.parser.add_argument
         # Controller config
         paa('--furl-file',
             type=unicode, dest='Global.furl_file',
             help='The full location of the file containing the FURL of the '
             'controller. If this is not given, the FURL file must be in the '
             'security directory of the cluster directory.  This location is '
             'resolved using the --profile and --app-dir options.',
             metavar='Global.furl_file')
         # MPI
         paa('--mpi',
             type=str, dest='MPI.use',
             help='How to enable MPI (mpi4py, pytrilinos, or empty string to disable).',
             metavar='MPI.use')
         # Global config
         paa('--log-to-file',
             action='store_true', dest='Global.log_to_file',
             help='Log to a file in the log directory (default is stdout)')
 
 
 #-----------------------------------------------------------------------------
 # Main application
 #-----------------------------------------------------------------------------
 
 
 class IPEngineApp(ApplicationWithClusterDir):
 
     name = u'ipengine'
     description = _description
     command_line_loader = IPEngineAppConfigLoader
-    config_file_name = default_config_file_name
+    default_config_file_name = default_config_file_name
     auto_create_cluster_dir = True
 
     def create_default_config(self):
         super(IPEngineApp, self).create_default_config()
 
         # The engine should not clean logs as we don't want to remove the
         # active log files of other running engines.
         self.default_config.Global.clean_logs = False
 
         # Global config attributes
         self.default_config.Global.exec_lines = []
         self.default_config.Global.shell_class = 'IPython.kernel.core.interpreter.Interpreter'
 
         # Configuration related to the controller
         # This must match the filename (path not included) that the controller
         # used for the FURL file.
         self.default_config.Global.furl_file_name = u'ipcontroller-engine.furl'
         # If given, this is the actual location of the controller's FURL file.
         # If not, this is computed using the profile, app_dir and furl_file_name
         self.default_config.Global.furl_file = u''
 
         # The max number of connection attemps and the initial delay between
         # those attemps.
         self.default_config.Global.connect_delay = 0.1
         self.default_config.Global.connect_max_tries = 15
 
         # MPI related config attributes
         self.default_config.MPI.use = ''
         self.default_config.MPI.mpi4py = mpi4py_init
         self.default_config.MPI.pytrilinos = pytrilinos_init
 
     def post_load_command_line_config(self):
         pass
 
     def pre_construct(self):
         super(IPEngineApp, self).pre_construct()
         self.find_cont_furl_file()
 
     def find_cont_furl_file(self):
         """Set the furl file.
 
         Here we don't try to actually see if it exists for is valid as that
         is hadled by the connection logic.
         """
         config = self.master_config
         # Find the actual controller FURL file
         if not config.Global.furl_file:
             try_this = os.path.join(
                 config.Global.cluster_dir, 
                 config.Global.security_dir,
                 config.Global.furl_file_name
             )
             config.Global.furl_file = try_this
 
     def construct(self):
         # This is the working dir by now.
         sys.path.insert(0, '')
 
         self.start_mpi()
         self.start_logging()
 
         # Create the underlying shell class and EngineService
         shell_class = import_item(self.master_config.Global.shell_class)
         self.engine_service = EngineService(shell_class, mpi=mpi)
 
         self.exec_lines()
 
         # Create the service hierarchy
         self.main_service = service.MultiService()
         self.engine_service.setServiceParent(self.main_service)
         self.tub_service = Tub()
         self.tub_service.setServiceParent(self.main_service)
         # This needs to be called before the connection is initiated
         self.main_service.startService()
 
         # This initiates the connection to the controller and calls
         # register_engine to tell the controller we are ready to do work
         self.engine_connector = EngineConnector(self.tub_service)
 
         log.msg("Using furl file: %s" % self.master_config.Global.furl_file)
 
         reactor.callWhenRunning(self.call_connect)
 
     def call_connect(self):
         d = self.engine_connector.connect_to_controller(
             self.engine_service, 
             self.master_config.Global.furl_file,
             self.master_config.Global.connect_delay,
             self.master_config.Global.connect_max_tries
         )
 
         def handle_error(f):
             log.msg('Error connecting to controller. This usually means that '
             'i) the controller was not started, ii) a firewall was blocking '
             'the engine from connecting to the controller or iii) the engine '
             ' was not pointed at the right FURL file:')
             log.msg(f.getErrorMessage())
             reactor.callLater(0.1, reactor.stop)
 
         d.addErrback(handle_error)
 
     def start_mpi(self):
         global mpi
         mpikey = self.master_config.MPI.use
         mpi_import_statement = self.master_config.MPI.get(mpikey, None)
         if mpi_import_statement is not None:
             try:
                 self.log.info("Initializing MPI:")
                 self.log.info(mpi_import_statement)
                 exec mpi_import_statement in globals()
             except:
                 mpi = None
         else:
             mpi = None
 
     def exec_lines(self):
         for line in self.master_config.Global.exec_lines:
             try:
                 log.msg("Executing statement: '%s'" % line)
                 self.engine_service.execute(line)
             except:
                 log.msg("Error executing statement: %s" % line)
 
     def start_app(self):
         reactor.run()
 
 
 def launch_new_instance():
     """Create and run the IPython controller"""
     app = IPEngineApp()
     app.start()
 
 
 if __name__ == '__main__':
     launch_new_instance()