diff --git a/IPython/__init__.py b/IPython/__init__.py index 90d7565..759f958 100755 --- a/IPython/__init__.py +++ b/IPython/__init__.py @@ -30,14 +30,14 @@ if sys.version[0:3] < '2.5': # Make it easy to import extensions - they are always directly on pythonpath. -# Therefore, non-IPython modules can be added to extensions directory +# Therefore, non-IPython modules can be added to extensions directory. +# This should probably be in ipapp.py. sys.path.append(os.path.join(os.path.dirname(__file__), "extensions")) #----------------------------------------------------------------------------- # Setup the top level names #----------------------------------------------------------------------------- -# In some cases, these are causing circular imports. from .config.loader import Config from .core import release from .core.application import Application diff --git a/IPython/config/default/kernel_config.py b/IPython/config/default/kernel_config.py deleted file mode 100644 index 0eec0dd..0000000 --- a/IPython/config/default/kernel_config.py +++ /dev/null @@ -1,62 +0,0 @@ -from os.path import join -pjoin = join - -from IPython.utils.genutils import get_ipython_dir, get_security_dir -security_dir = get_security_dir() - - -ENGINE_LOGFILE = '' - -ENGINE_FURL_FILE = 'ipcontroller-engine.furl' - -MPI_CONFIG_MPI4PY = """from mpi4py import MPI as mpi -mpi.size = mpi.COMM_WORLD.Get_size() -mpi.rank = mpi.COMM_WORLD.Get_rank() -""" - -MPI_CONFIG_PYTRILINOS = """from PyTrilinos import Epetra -class SimpleStruct: -pass -mpi = SimpleStruct() -mpi.rank = 0 -mpi.size = 0 -""" - -MPI_DEFAULT = '' - -CONTROLLER_LOGFILE = '' -CONTROLLER_IMPORT_STATEMENT = '' -CONTROLLER_REUSE_FURLS = False - -ENGINE_TUB_IP = '' -ENGINE_TUB_PORT = 0 -ENGINE_TUB_LOCATION = '' -ENGINE_TUB_SECURE = True -ENGINE_TUB_CERT_FILE = 'ipcontroller-engine.pem' -ENGINE_FC_INTERFACE = 'IPython.kernel.enginefc.IFCControllerBase' -ENGINE_FURL_FILE = 'ipcontroller-engine.furl' - -CONTROLLER_INTERFACES = dict( - TASK = dict( - CONTROLLER_INTERFACE = 'IPython.kernel.task.ITaskController', - FC_INTERFACE = 'IPython.kernel.taskfc.IFCTaskController', - FURL_FILE = pjoin(security_dir, 'ipcontroller-tc.furl') - ), - MULTIENGINE = dict( - CONTROLLER_INTERFACE = 'IPython.kernel.multiengine.IMultiEngine', - FC_INTERFACE = 'IPython.kernel.multienginefc.IFCSynchronousMultiEngine', - FURL_FILE = pjoin(security_dir, 'ipcontroller-mec.furl') - ) -) - -CLIENT_TUB_IP = '' -CLIENT_TUB_PORT = 0 -CLIENT_TUB_LOCATION = '' -CLIENT_TUB_SECURE = True -CLIENT_TUB_CERT_FILE = 'ipcontroller-client.pem' - -CLIENT_INTERFACES = dict( - TASK = dict(FURL_FILE = 'ipcontroller-tc.furl'), - MULTIENGINE = dict(FURLFILE='ipcontroller-mec.furl') -) - diff --git a/IPython/config/loader.py b/IPython/config/loader.py index 9198c37..df79679 100644 --- a/IPython/config/loader.py +++ b/IPython/config/loader.py @@ -23,7 +23,7 @@ import os import sys from IPython.external import argparse -from IPython.utils.genutils import filefind +from IPython.utils.path import filefind #----------------------------------------------------------------------------- # Exceptions @@ -40,6 +40,7 @@ class ConfigLoaderError(ConfigError): #----------------------------------------------------------------------------- # Argparse fix #----------------------------------------------------------------------------- + # Unfortunately argparse by default prints help messages to stderr instead of # stdout. This makes it annoying to capture long help screens at the command # line, since one must know how to pipe stderr, which many users don't know how @@ -200,10 +201,13 @@ class ConfigLoader(object): self.config = Config() def load_config(self): - """Load a config from somewhere, return a Struct. + """Load a config from somewhere, return a :class:`Config` instance. Usually, this will cause self.config to be set and then returned. + However, in most cases, :meth:`ConfigLoader.clear` should be called + to erase any previous state. """ + self.clear() return self.config @@ -242,6 +246,7 @@ class PyFileConfigLoader(FileConfigLoader): def load_config(self): """Load the config from a file and return it as a Struct.""" + self.clear() self._find_file() self._read_file_as_dict() self._convert_to_config() @@ -292,20 +297,10 @@ class CommandLineConfigLoader(ConfigLoader): """ -class __NoConfigDefault(object): pass -NoConfigDefault = __NoConfigDefault() - - class ArgParseConfigLoader(CommandLineConfigLoader): - #: Global default for arguments (see argparse docs for details) - argument_default = NoConfigDefault - - def __init__(self, argv=None, arguments=(), *args, **kw): - """Create a config loader for use with argparse. - With the exception of ``argv`` and ``arguments``, other args and kwargs - arguments here are passed onto the constructor of - :class:`argparse.ArgumentParser`. + def __init__(self, argv=None, *parser_args, **parser_kw): + """Create a config loader for use with argparse. Parameters ---------- @@ -314,19 +309,22 @@ class ArgParseConfigLoader(CommandLineConfigLoader): If given, used to read command-line arguments from, otherwise sys.argv[1:] is used. - arguments : optional, tuple - Description of valid command-line arguments, to be called in sequence - with parser.add_argument() to configure the parser. + parser_args : tuple + A tuple of positional arguments that will be passed to the + constructor of :class:`argparse.ArgumentParser`. + + parser_kw : dict + A tuple of keyword arguments that will be passed to the + constructor of :class:`argparse.ArgumentParser`. """ super(CommandLineConfigLoader, self).__init__() if argv == None: argv = sys.argv[1:] self.argv = argv - self.arguments = arguments - self.args = args - kwargs = dict(argument_default=self.argument_default) - kwargs.update(kw) - self.kw = kwargs + self.parser_args = parser_args + kwargs = dict(argument_default=argparse.SUPPRESS) + kwargs.update(parser_kw) + self.parser_kw = kwargs def load_config(self, args=None): """Parse command line arguments and return as a Struct. @@ -335,10 +333,10 @@ class ArgParseConfigLoader(CommandLineConfigLoader): ---------- args : optional, list - If given, a list with the structure of sys.argv[1:] to parse arguments - from. If not given, the instance's self.argv attribute (given at - construction time) is used.""" - + If given, a list with the structure of sys.argv[1:] to parse + arguments from. If not given, the instance's self.argv attribute + (given at construction time) is used.""" + self.clear() if args is None: args = self.argv self._create_parser() @@ -353,17 +351,11 @@ class ArgParseConfigLoader(CommandLineConfigLoader): return [] def _create_parser(self): - self.parser = ArgumentParser(*self.args, **self.kw) + self.parser = ArgumentParser(*self.parser_args, **self.parser_kw) self._add_arguments() - self._add_other_arguments() def _add_arguments(self): - for argument in self.arguments: - self.parser.add_argument(*argument[0],**argument[1]) - - def _add_other_arguments(self): - """Meant for subclasses to add their own arguments.""" - pass + raise NotImplementedError("subclasses must implement _add_arguments") def _parse_args(self, args): """self.parser->self.parsed_data""" @@ -372,6 +364,7 @@ class ArgParseConfigLoader(CommandLineConfigLoader): def _convert_to_config(self): """self.parsed_data->self.config""" for k, v in vars(self.parsed_data).items(): - if v is not NoConfigDefault: - exec_str = 'self.config.' + k + '= v' - exec exec_str in locals(), globals() + exec_str = 'self.config.' + k + '= v' + exec exec_str in locals(), globals() + + diff --git a/IPython/config/tests/test_loader.py b/IPython/config/tests/test_loader.py index 487db8b..390f1b4 100755 --- a/IPython/config/tests/test_loader.py +++ b/IPython/config/tests/test_loader.py @@ -61,35 +61,38 @@ class TestPyFileCL(TestCase): self.assertEquals(config.Foo.Bam.value, range(10)) self.assertEquals(config.D.C.value, 'hi there') +class MyLoader1(ArgParseConfigLoader): + def _add_arguments(self): + p = self.parser + p.add_argument('-f', '--foo', dest='Global.foo', type=str) + p.add_argument('-b', dest='MyClass.bar', type=int) + p.add_argument('-n', dest='n', action='store_true') + p.add_argument('Global.bam', type=str) + +class MyLoader2(ArgParseConfigLoader): + def _add_arguments(self): + subparsers = self.parser.add_subparsers(dest='subparser_name') + subparser1 = subparsers.add_parser('1') + subparser1.add_argument('-x',dest='Global.x') + subparser2 = subparsers.add_parser('2') + subparser2.add_argument('y') class TestArgParseCL(TestCase): def test_basic(self): - - arguments = ( - (('-f','--foo'), dict(dest='Global.foo', type=str)), - (('-b',), dict(dest='MyClass.bar', type=int)), - (('-n',), dict(dest='n', action='store_true')), - (('Global.bam',), dict(type=str)) - ) - cl = ArgParseConfigLoader(arguments=arguments) + cl = MyLoader1() config = cl.load_config('-f hi -b 10 -n wow'.split()) self.assertEquals(config.Global.foo, 'hi') self.assertEquals(config.MyClass.bar, 10) self.assertEquals(config.n, True) self.assertEquals(config.Global.bam, 'wow') + config = cl.load_config(['wow']) + self.assertEquals(config.keys(), ['Global']) + self.assertEquals(config.Global.keys(), ['bam']) + self.assertEquals(config.Global.bam, 'wow') def test_add_arguments(self): - - class MyLoader(ArgParseConfigLoader): - def _add_arguments(self): - subparsers = self.parser.add_subparsers(dest='subparser_name') - subparser1 = subparsers.add_parser('1') - subparser1.add_argument('-x',dest='Global.x') - subparser2 = subparsers.add_parser('2') - subparser2.add_argument('y') - - cl = MyLoader() + cl = MyLoader2() config = cl.load_config('2 frobble'.split()) self.assertEquals(config.subparser_name, '2') self.assertEquals(config.y, 'frobble') @@ -97,6 +100,15 @@ class TestArgParseCL(TestCase): self.assertEquals(config.subparser_name, '1') self.assertEquals(config.Global.x, 'frobble') + def test_argv(self): + cl = MyLoader1(argv='-f hi -b 10 -n wow'.split()) + config = cl.load_config() + self.assertEquals(config.Global.foo, 'hi') + self.assertEquals(config.MyClass.bar, 10) + self.assertEquals(config.n, True) + self.assertEquals(config.Global.bam, 'wow') + + class TestConfig(TestCase): def test_setget(self): diff --git a/IPython/core/alias.py b/IPython/core/alias.py index 3dbb8cd..1afe854 100644 --- a/IPython/core/alias.py +++ b/IPython/core/alias.py @@ -28,9 +28,9 @@ import sys from IPython.core.component import Component from IPython.core.splitinput import split_user_input -from IPython.utils.traitlets import CBool, List, Instance -from IPython.utils.genutils import error +from IPython.utils.traitlets import List from IPython.utils.autoattr import auto_attr +from IPython.utils.warn import warn, error #----------------------------------------------------------------------------- # Utilities diff --git a/IPython/core/application.py b/IPython/core/application.py index 9a88ccc..d53fc6d 100644 --- a/IPython/core/application.py +++ b/IPython/core/application.py @@ -33,7 +33,7 @@ import os import sys from IPython.core import release, crashhandler -from IPython.utils.genutils import get_ipython_dir, get_ipython_package_dir +from IPython.utils.path import get_ipython_dir, get_ipython_package_dir from IPython.config.loader import ( PyFileConfigLoader, ArgParseConfigLoader, @@ -48,108 +48,86 @@ class ApplicationError(Exception): pass -app_cl_args = ( - (('--ipython-dir', ), dict( - dest='Global.ipython_dir',type=unicode, - help= - """Set to override default location of the IPython directory - IPYTHON_DIR, stored as Global.ipython_dir. This can also be specified - through the environment variable IPYTHON_DIR.""", - metavar='Global.ipython_dir') ), - (('-p', '--profile',), dict( - dest='Global.profile',type=unicode, - help= - """The string name of the ipython profile to be used. Assume that your - config file is ipython_config-.py (looks in current dir first, - then in IPYTHON_DIR). This is a quick way to keep and load multiple - config files for different tasks, especially if include your basic one - in your more specialized ones. You can keep a basic - IPYTHON_DIR/ipython_config.py file and then have other 'profiles' which - include this one and load extra things for particular tasks.""", - metavar='Global.profile') ), - (('--log-level',), dict( - dest="Global.log_level",type=int, - help='Set the log level (0,10,20,30,40,50). Default is 30.', - metavar='Global.log_level')), - (('--config-file',), dict( - dest='Global.config_file',type=unicode, - help= - """Set the config file name to override default. Normally IPython - loads ipython_config.py (from current directory) or - IPYTHON_DIR/ipython_config.py. If the loading of your config file - fails, IPython starts with a bare bones configuration (no modules - loaded at all).""", - metavar='Global.config_file')), - ) +class BaseAppConfigLoader(ArgParseConfigLoader): + """Default command line options for IPython based applications.""" + + def _add_ipython_dir(self, parser): + """Add the --ipython-dir option to the parser.""" + paa = parser.add_argument + paa('--ipython-dir', + dest='Global.ipython_dir',type=unicode, + help= + """Set to override default location of the IPython directory + IPYTHON_DIR, stored as Global.ipython_dir. This can also be + specified through the environment variable IPYTHON_DIR.""", + metavar='Global.ipython_dir') + + def _add_log_level(self, parser): + """Add the --log-level option to the parser.""" + paa = parser.add_argument + paa('--log-level', + dest="Global.log_level",type=int, + help='Set the log level (0,10,20,30,40,50). Default is 30.', + metavar='Global.log_level') + + def _add_arguments(self): + self._add_ipython_dir(self.parser) + self._add_log_level(self.parser) + class Application(object): """Load a config, construct components and set them running. - The configuration of an application can be done via four different Config - objects, which are loaded and ultimately merged into a single one used from - that point on by the app. These are: + The configuration of an application can be done via three different Config + objects, which are loaded and ultimately merged into a single one used + from that point on by the app. These are: 1. default_config: internal defaults, implemented in code. 2. file_config: read from the filesystem. 3. command_line_config: read from the system's command line flags. - 4. constructor_config: passed parametrically to the constructor. During initialization, 3 is actually read before 2, since at the command-line one may override the location of the file to be read. But the above is the order in which the merge is made. - - There is a final config object can be created and passed to the - constructor: override_config. If it exists, this completely overrides the - configs 2-4 above (the default is still used to ensure that all needed - fields at least are created). This makes it easier to create - parametrically (e.g. in testing or sphinx plugins) objects with a known - configuration, that are unaffected by whatever arguments may be present in - sys.argv or files in the user's various directories. """ name = u'ipython' description = 'IPython: an enhanced interactive Python shell.' - #: usage message printed by argparse. If None, auto-generate + #: Usage message printed by argparse. If None, auto-generate usage = None - config_file_name = u'ipython_config.py' - #: Track the default and actual separately because some messages are - #: only printed if we aren't using the default. - default_config_file_name = config_file_name + #: The command line config loader. Subclass of ArgParseConfigLoader. + command_line_loader = BaseAppConfigLoader + #: The name of the config file to load, determined at runtime + config_file_name = None + #: The name of the default config file. Track separately from the actual + #: name because some logic happens only if we aren't using the default. + default_config_file_name = u'ipython_config.py' default_log_level = logging.WARN #: Set by --profile option profile_name = None #: User's ipython directory, typically ~/.ipython/ ipython_dir = None - #: internal defaults, implemented in code. + #: Internal defaults, implemented in code. default_config = None - #: read from the filesystem + #: Read from the filesystem. file_config = None - #: read from the system's command line flags + #: Read from the system's command line flags. command_line_config = None - #: passed parametrically to the constructor. - constructor_config = None - #: final override, if given supercedes file/command/constructor configs - override_config = None + #: The final config that will be passed to the component. + master_config = None #: A reference to the argv to be used (typically ends up being sys.argv[1:]) argv = None - #: Default command line arguments. Subclasses should create a new tuple - #: that *includes* these. - cl_arguments = app_cl_args - #: extra arguments computed by the command-line loader extra_args = None + #: The class to use as the crash handler. + crash_handler_class = crashhandler.CrashHandler # Private attributes _exiting = False _initialized = False - # Class choices for things that will be instantiated at runtime. - _CrashHandler = crashhandler.CrashHandler - - def __init__(self, argv=None, constructor_config=None, override_config=None): + def __init__(self, argv=None): self.argv = sys.argv[1:] if argv is None else argv - self.constructor_config = constructor_config - self.override_config = override_config self.init_logger() def init_logger(self): @@ -194,13 +172,12 @@ class Application(object): self.log_default_config() self.set_default_config_log_level() - if self.override_config is None: - # Command-line config - self.pre_load_command_line_config() - self.load_command_line_config() - self.set_command_line_config_log_level() - self.post_load_command_line_config() - self.log_command_line_config() + # Command-line config + self.pre_load_command_line_config() + self.load_command_line_config() + self.set_command_line_config_log_level() + self.post_load_command_line_config() + self.log_command_line_config() # Find resources needed for filesystem access, using information from # the above two @@ -209,13 +186,12 @@ class Application(object): self.find_config_file_name() self.find_config_file_paths() - if self.override_config is None: - # File-based config - self.pre_load_file_config() - self.load_file_config() - self.set_file_config_log_level() - self.post_load_file_config() - self.log_file_config() + # File-based config + self.pre_load_file_config() + self.load_file_config() + self.set_file_config_log_level() + self.post_load_file_config() + self.log_file_config() # Merge all config objects into a single one the app can then use self.merge_configs() @@ -240,7 +216,7 @@ class Application(object): def create_crash_handler(self): """Create a crash handler, typically setting sys.excepthook to it.""" - self.crash_handler = self._CrashHandler(self, self.name) + self.crash_handler = self.crash_handler_class(self) sys.excepthook = self.crash_handler def create_default_config(self): @@ -270,11 +246,12 @@ class Application(object): def create_command_line_config(self): """Create and return a command line config loader.""" - return ArgParseConfigLoader(self.argv, self.cl_arguments, - description=self.description, - version=release.version, - usage=self.usage, - ) + return self.command_line_loader( + self.argv, + description=self.description, + version=release.version, + usage=self.usage + ) def pre_load_command_line_config(self): """Do actions just before loading the command line config.""" @@ -338,18 +315,22 @@ class Application(object): If a profile has been set at the command line, this will resolve it. """ - try: self.config_file_name = self.command_line_config.Global.config_file except AttributeError: pass + else: + return try: self.profile_name = self.command_line_config.Global.profile except AttributeError: - pass + # Just use the default as there is no profile + self.config_file_name = self.default_config_file_name else: - name_parts = self.config_file_name.split('.') + # Use the default config file name and profile name if set + # to determine the used config file name. + name_parts = self.default_config_file_name.split('.') name_parts.insert(1, u'_' + self.profile_name + u'.') self.config_file_name = ''.join(name_parts) @@ -418,13 +399,9 @@ class Application(object): """Merge the default, command line and file config objects.""" config = Config() config._merge(self.default_config) - if self.override_config is None: - config._merge(self.file_config) - config._merge(self.command_line_config) - if self.constructor_config is not None: - config._merge(self.constructor_config) - else: - config._merge(self.override_config) + config._merge(self.file_config) + config._merge(self.command_line_config) + # XXX fperez - propose to Brian we rename master_config to simply # config, I think this is going to be heavily used in examples and # application code and the name is shorter/easier to find/remember. @@ -456,15 +433,6 @@ class Application(object): # Utility methods #------------------------------------------------------------------------- - def abort(self): - """Abort the starting of the application.""" - if self._exiting: - pass - else: - self.log.critical("Aborting application: %s" % self.name, exc_info=True) - self._exiting = True - sys.exit(1) - def exit(self, exit_status=0): if self._exiting: pass @@ -473,17 +441,13 @@ class Application(object): self._exiting = True sys.exit(exit_status) - def attempt(self, func, action='abort'): + def attempt(self, func): try: func() except SystemExit: raise except: - if action == 'abort': - self.log.critical("Aborting application: %s" % self.name, - exc_info=True) - self.abort() - raise - elif action == 'exit': - self.exit(0) + self.log.critical("Aborting application: %s" % self.name, + exc_info=True) + self.exit(0) diff --git a/IPython/core/completer.py b/IPython/core/completer.py index 049e982..07fd037 100644 --- a/IPython/core/completer.py +++ b/IPython/core/completer.py @@ -69,19 +69,20 @@ used, and this module (and the readline module) are silently inactive. import __builtin__ import __main__ import glob +import inspect import itertools import keyword import os import re import shlex import sys -import types -import IPython.utils.rlineimpl as readline from IPython.core.error import TryNext from IPython.core.prefilter import ESC_MAGIC from IPython.utils import generics -from IPython.utils.genutils import debugx, dir2 +from IPython.utils.frame import debugx +from IPython.utils.dir2 import dir2 +import IPython.utils.rlineimpl as readline #----------------------------------------------------------------------------- # Globals @@ -216,7 +217,6 @@ class Completer: with a __getattr__ hook is evaluated. """ - import re #print 'Completer->attr_matches, txt=%r' % text # dbg # Another option, seems to work great. Catches things like ''. diff --git a/IPython/core/component.py b/IPython/core/component.py index 42c6118..1ee9509 100644 --- a/IPython/core/component.py +++ b/IPython/core/component.py @@ -27,7 +27,7 @@ from weakref import WeakValueDictionary from IPython.utils.importstring import import_item from IPython.config.loader import Config from IPython.utils.traitlets import ( - HasTraitlets, TraitletError, MetaHasTraitlets, Instance, This + HasTraitlets, MetaHasTraitlets, Instance, This ) diff --git a/IPython/core/crashhandler.py b/IPython/core/crashhandler.py index f5c80b8..5c9c3ab 100644 --- a/IPython/core/crashhandler.py +++ b/IPython/core/crashhandler.py @@ -1,124 +1,113 @@ -# -*- coding: utf-8 -*- +# encoding: utf-8 """sys.excepthook for IPython itself, leaves a detailed report on disk. +Authors: -Authors -------- -- Fernando Perez +* Fernando Perez +* Brian E. Granger """ -#***************************************************************************** -# Copyright (C) 2008-2009 The IPython Development Team -# Copyright (C) 2001-2007 Fernando Perez. +#----------------------------------------------------------------------------- +# Copyright (C) 2001-2007 Fernando Perez. +# Copyright (C) 2008-2010 The IPython Development Team # # Distributed under the terms of the BSD License. The full license is in # the file COPYING, distributed as part of this software. -#***************************************************************************** +#----------------------------------------------------------------------------- -#**************************************************************************** -# Required modules +#----------------------------------------------------------------------------- +# Imports +#----------------------------------------------------------------------------- -# From the standard library import os import sys from pprint import pformat -# Our own -from IPython.core import release from IPython.core import ultratb -from IPython.utils.genutils import sys_info - from IPython.external.Itpl import itpl +from IPython.utils.sysinfo import sys_info -#**************************************************************************** +#----------------------------------------------------------------------------- +# Code +#----------------------------------------------------------------------------- -class CrashHandler(object): - """Customizable crash handlers for IPython-based systems. +# Template for the user message. +_default_message_template = """\ +Oops, $self.app_name crashed. We do our best to make it stable, but... + +A crash report was automatically generated with the following information: + - A verbatim copy of the crash traceback. + - A copy of your input history during this session. + - Data on your current $self.app_name configuration. - Instances of this class provide a __call__ method which can be used as a - sys.excepthook, i.e., the __call__ signature is: +It was left in the file named: +\t'$self.crash_report_fname' +If you can email this file to the developers, the information in it will help +them in understanding and correcting the problem. - def __call__(self,etype, evalue, etb) +You can mail it to: $self.contact_name at $self.contact_email +with the subject '$self.app_name Crash Report'. - """ +If you want to do it now, the following command will work (under Unix): +mail -s '$self.app_name Crash Report' $self.contact_email < $self.crash_report_fname + +To ensure accurate tracking of this issue, please file a report about it at: +$self.bug_tracker +""" - def __init__(self,app, app_name, contact_name=None, contact_email=None, - bug_tracker=None, crash_report_fname='CrashReport.txt', - show_crash_traceback=True, call_pdb=False): - """New crash handler. - Inputs: +class CrashHandler(object): + """Customizable crash handlers for IPython applications. - - app: a running application instance, which will be queried at crash - time for internal information. + Instances of this class provide a :meth:`__call__` method which can be + used as a ``sys.excepthook``. The :meth:`__call__` signature is:: - - app_name: a string containing the name of your application. + def __call__(self, etype, evalue, etb) + """ - - contact_name: a string with the name of the person to contact. + message_template = _default_message_template - - contact_email: a string with the email address of the contact. + def __init__(self, app, contact_name=None, contact_email=None, + bug_tracker=None, show_crash_traceback=True, call_pdb=False): + """Create a new crash handler - - bug_tracker: a string with the URL for your project's bug tracker. + Parameters + ---------- + app : Application + A running :class:`Application` instance, which will be queried at + crash time for internal information. - - crash_report_fname: a string with the filename for the crash report - to be saved in. These reports are left in the ipython user directory - as determined by the running IPython instance. + contact_name : str + A string with the name of the person to contact. - Optional inputs: - - - show_crash_traceback(True): if false, don't print the crash - traceback on stderr, only generate the on-disk report + contact_email : str + A string with the email address of the contact. + bug_tracker : str + A string with the URL for your project's bug tracker. + + show_crash_traceback : bool + If false, don't print the crash traceback on stderr, only generate + the on-disk report Non-argument instance attributes: - These instances contain some non-argument attributes which allow for - further customization of the crash handler's behavior. Please see the + These instances contain some non-argument attributes which allow for + further customization of the crash handler's behavior. Please see the source for further details. """ - - # apply args into instance self.app = app - self.app_name = app_name + self.app_name = self.app.name self.contact_name = contact_name self.contact_email = contact_email self.bug_tracker = bug_tracker - self.crash_report_fname = crash_report_fname + self.crash_report_fname = "Crash_report_%s.txt" % self.app_name self.show_crash_traceback = show_crash_traceback self.section_sep = '\n\n'+'*'*75+'\n\n' self.call_pdb = call_pdb #self.call_pdb = True # dbg - - # Hardcoded defaults, which can be overridden either by subclasses or - # at runtime for the instance. - - # Template for the user message. Subclasses which completely override - # this, or user apps, can modify it to suit their tastes. It gets - # expanded using itpl, so calls of the kind $self.foo are valid. - self.user_message_template = """ -Oops, $self.app_name crashed. We do our best to make it stable, but... -A crash report was automatically generated with the following information: - - A verbatim copy of the crash traceback. - - A copy of your input history during this session. - - Data on your current $self.app_name configuration. - -It was left in the file named: -\t'$self.crash_report_fname' -If you can email this file to the developers, the information in it will help -them in understanding and correcting the problem. - -You can mail it to: $self.contact_name at $self.contact_email -with the subject '$self.app_name Crash Report'. - -If you want to do it now, the following command will work (under Unix): -mail -s '$self.app_name Crash Report' $self.contact_email < $self.crash_report_fname - -To ensure accurate tracking of this issue, please file a report about it at: -$self.bug_tracker -""" - - def __call__(self,etype, evalue, etb): + def __call__(self, etype, evalue, etb): """Handle an exception, call for compatible with sys.excepthook""" # Report tracebacks shouldn't use color in general (safer for users) @@ -137,10 +126,11 @@ $self.bug_tracker # write the report filename into the instance dict so it can get # properly expanded out in the user message template self.crash_report_fname = report_name - TBhandler = ultratb.VerboseTB(color_scheme=color_scheme, - long_header=1, - call_pdb=self.call_pdb, - ) + TBhandler = ultratb.VerboseTB( + color_scheme=color_scheme, + long_header=1, + call_pdb=self.call_pdb, + ) if self.call_pdb: TBhandler(etype,evalue,etb) return @@ -159,7 +149,7 @@ $self.bug_tracker return # Inform user on stderr of what happened - msg = itpl('\n'+'*'*70+'\n'+self.user_message_template) + msg = itpl('\n'+'*'*70+'\n'+self.message_template) print >> sys.stderr, msg # Construct report on disk @@ -178,7 +168,9 @@ $self.bug_tracker try: config = pformat(self.app.config) - rpt_add(sec_sep+'Current user configuration structure:\n\n') + rpt_add(sec_sep) + rpt_add('Application name: %s\n\n' % self.app_name) + rpt_add('Current user configuration structure:\n\n') rpt_add(config) except: pass @@ -186,38 +178,3 @@ $self.bug_tracker return ''.join(report) - -class IPythonCrashHandler(CrashHandler): - """sys.excepthook for IPython itself, leaves a detailed report on disk.""" - - def __init__(self, app, app_name='IPython'): - - # Set here which of the IPython authors should be listed as contact - AUTHOR_CONTACT = 'Fernando' - - # Set argument defaults - bug_tracker = 'https://bugs.launchpad.net/ipython/+filebug' - contact_name,contact_email = release.authors[AUTHOR_CONTACT][:2] - crash_report_fname = 'IPython_crash_report.txt' - # Call parent constructor - CrashHandler.__init__(self,app,app_name,contact_name,contact_email, - bug_tracker,crash_report_fname) - - def make_report(self,traceback): - """Return a string containing a crash report.""" - - sec_sep = self.section_sep - # Start with parent report - report = [super(IPythonCrashHandler, self).make_report(traceback)] - # Add interactive-specific info we may have - rpt_add = report.append - try: - rpt_add(sec_sep+"History of session input:") - for line in self.app.shell.user_ns['_ih']: - rpt_add(line) - rpt_add('\n*** Last line of input (may not be in above history):\n') - rpt_add(self.app.shell._last_input_line+'\n') - except: - pass - - return ''.join(report) diff --git a/IPython/core/debugger.py b/IPython/core/debugger.py index 45fe177..151a4b1 100644 --- a/IPython/core/debugger.py +++ b/IPython/core/debugger.py @@ -26,15 +26,13 @@ http://www.python.org/2.2.3/license.html""" #***************************************************************************** import bdb -import cmd import linecache -import os import sys from IPython.utils import PyColorize from IPython.core import ipapi from IPython.utils import coloransi -from IPython.utils.genutils import Term +from IPython.utils.io import Term from IPython.core.excolors import exception_colors # See if we can use pydb. diff --git a/IPython/core/display_trap.py b/IPython/core/display_trap.py index c0f0834..d5e5e83 100644 --- a/IPython/core/display_trap.py +++ b/IPython/core/display_trap.py @@ -24,8 +24,6 @@ import sys from IPython.core.component import Component -from IPython.utils.autoattr import auto_attr - #----------------------------------------------------------------------------- # Classes and functions #----------------------------------------------------------------------------- diff --git a/IPython/core/embed.py b/IPython/core/embed.py index a43342a..16d6c78 100644 --- a/IPython/core/embed.py +++ b/IPython/core/embed.py @@ -24,6 +24,7 @@ Notes #----------------------------------------------------------------------------- from __future__ import with_statement +import __main__ import sys from contextlib import nested @@ -33,7 +34,7 @@ from IPython.core.iplib import InteractiveShell from IPython.core.ipapp import load_default_config from IPython.utils.traitlets import Bool, Str, CBool -from IPython.utils.genutils import ask_yes_no +from IPython.utils.io import ask_yes_no #----------------------------------------------------------------------------- diff --git a/IPython/core/excolors.py b/IPython/core/excolors.py index f36186b..14451d5 100644 --- a/IPython/core/excolors.py +++ b/IPython/core/excolors.py @@ -10,8 +10,6 @@ Color schemes for exception handling code in IPython. # the file COPYING, distributed as part of this software. #***************************************************************************** -#**************************************************************************** -# Required modules from IPython.utils.coloransi import ColorSchemeTable, TermColors, ColorScheme def exception_colors(): diff --git a/IPython/core/history.py b/IPython/core/history.py index a508f9c..e21a197 100644 --- a/IPython/core/history.py +++ b/IPython/core/history.py @@ -5,7 +5,8 @@ import fnmatch import os -from IPython.utils.genutils import Term, ask_yes_no, warn +from IPython.utils.io import Term, ask_yes_no +from IPython.utils.warn import warn from IPython.core import ipapi def magic_history(self, parameter_s = ''): diff --git a/IPython/core/hooks.py b/IPython/core/hooks.py index 79eb5ba..97990c8 100644 --- a/IPython/core/hooks.py +++ b/IPython/core/hooks.py @@ -43,9 +43,12 @@ somewhere in your configuration files or ipython command line. import os, bisect import sys -from IPython.utils.genutils import Term, shell + from pprint import PrettyPrinter +from IPython.utils.io import Term +from IPython.utils.process import shell + from IPython.core.error import TryNext # List here all the default hooks. For now it's just the editor functions diff --git a/IPython/core/ipapi.py b/IPython/core/ipapi.py index 8ee88a2..461aa5a 100644 --- a/IPython/core/ipapi.py +++ b/IPython/core/ipapi.py @@ -18,8 +18,6 @@ has been made into a component, this module will be sent to deathrow. # Imports #----------------------------------------------------------------------------- -from IPython.core.error import TryNext, UsageError, IPythonCoreError - #----------------------------------------------------------------------------- # Classes and functions #----------------------------------------------------------------------------- diff --git a/IPython/core/ipapp.py b/IPython/core/ipapp.py index 95dd2c1..efd42f9 100755 --- a/IPython/core/ipapp.py +++ b/IPython/core/ipapp.py @@ -21,374 +21,376 @@ Authors #----------------------------------------------------------------------------- # Imports #----------------------------------------------------------------------------- + from __future__ import absolute_import import logging import os import sys -from IPython.core import crashhandler -from IPython.core.application import Application +from IPython.core import release +from IPython.core.crashhandler import CrashHandler +from IPython.core.application import Application, BaseAppConfigLoader from IPython.core.iplib import InteractiveShell from IPython.config.loader import ( Config, - PyFileConfigLoader, -# NoConfigDefault, + PyFileConfigLoader ) from IPython.lib import inputhook -from IPython.utils.genutils import filefind, get_ipython_dir +from IPython.utils.path import filefind, get_ipython_dir from . import usage #----------------------------------------------------------------------------- # Globals, utilities and helpers #----------------------------------------------------------------------------- +#: The default config file name for this application. default_config_file_name = u'ipython_config.py' -cl_args = ( - (('--autocall',), dict( - type=int, dest='InteractiveShell.autocall', - help= - """Make IPython automatically call any callable object even if you - didn't type explicit parentheses. For example, 'str 43' becomes - 'str(43)' automatically. The value can be '0' to disable the feature, - '1' for 'smart' autocall, where it is not applied if there are no more - arguments on the line, and '2' for 'full' autocall, where all callable - objects are automatically called (even if no arguments are present). - The default is '1'.""", - metavar='InteractiveShell.autocall') - ), - (('--autoindent',), dict( - action='store_true', dest='InteractiveShell.autoindent', - help='Turn on autoindenting.') - ), - (('--no-autoindent',), dict( - action='store_false', dest='InteractiveShell.autoindent', - help='Turn off autoindenting.') - ), - (('--automagic',), dict( - action='store_true', dest='InteractiveShell.automagic', - help='Turn on the auto calling of magic commands.' - 'Type %%magic at the IPython prompt for more information.') - ), - (('--no-automagic',), dict( - action='store_false', dest='InteractiveShell.automagic', - help='Turn off the auto calling of magic commands.') - ), - (('--autoedit-syntax',), dict( - action='store_true', dest='InteractiveShell.autoedit_syntax', - help='Turn on auto editing of files with syntax errors.') - ), - (('--no-autoedit-syntax',), dict( - action='store_false', dest='InteractiveShell.autoedit_syntax', - help='Turn off auto editing of files with syntax errors.') - ), - (('--banner',), dict( - action='store_true', dest='Global.display_banner', - help='Display a banner upon starting IPython.') - ), - (('--no-banner',), dict( - action='store_false', dest='Global.display_banner', - help="Don't display a banner upon starting IPython.") - ), - (('--cache-size',), dict( - type=int, dest='InteractiveShell.cache_size', - help= - """Set the size of the output cache. The default is 1000, you can - change it permanently in your config file. Setting it to 0 completely - disables the caching system, and the minimum value accepted is 20 (if - you provide a value less than 20, it is reset to 0 and a warning is - issued). This limit is defined because otherwise you'll spend more - time re-flushing a too small cache than working. - """, - metavar='InteractiveShell.cache_size') - ), - (('--classic',), dict( - action='store_true', dest='Global.classic', - help="Gives IPython a similar feel to the classic Python prompt.") - ), - (('--colors',), dict( - type=str, dest='InteractiveShell.colors', - help="Set the color scheme (NoColor, Linux, and LightBG).", - metavar='InteractiveShell.colors') - ), - (('--color-info',), dict( - action='store_true', dest='InteractiveShell.color_info', - help= - """IPython can display information about objects via a set of func- - tions, and optionally can use colors for this, syntax highlighting - source code and various other elements. However, because this - information is passed through a pager (like 'less') and many pagers get - confused with color codes, this option is off by default. You can test - it and turn it on permanently in your ipython_config.py file if it - works for you. Test it and turn it on permanently if it works with - your system. The magic function %%color_info allows you to toggle this - inter- actively for testing.""" - ) - ), - (('--no-color-info',), dict( - action='store_false', dest='InteractiveShell.color_info', - help="Disable using colors for info related things.") - ), - (('--confirm-exit',), dict( - action='store_true', dest='InteractiveShell.confirm_exit', - help= - """Set to confirm when you try to exit IPython with an EOF (Control-D - in Unix, Control-Z/Enter in Windows). By typing 'exit', 'quit' or - '%%Exit', you can force a direct exit without any confirmation. - """ - ) - ), - (('--no-confirm-exit',), dict( - action='store_false', dest='InteractiveShell.confirm_exit', - help="Don't prompt the user when exiting.") - ), - (('--deep-reload',), dict( - action='store_true', dest='InteractiveShell.deep_reload', - help= - """Enable deep (recursive) reloading by default. IPython can use the - deep_reload module which reloads changes in modules recursively (it - replaces the reload() function, so you don't need to change anything to - use it). deep_reload() forces a full reload of modules whose code may - have changed, which the default reload() function does not. When - deep_reload is off, IPython will use the normal reload(), but - deep_reload will still be available as dreload(). This fea- ture is off - by default [which means that you have both normal reload() and - dreload()].""") - ), - (('--no-deep-reload',), dict( - action='store_false', dest='InteractiveShell.deep_reload', - help="Disable deep (recursive) reloading by default.") - ), - (('--editor',), dict( - type=str, dest='InteractiveShell.editor', - help="Set the editor used by IPython (default to $EDITOR/vi/notepad).", - metavar='InteractiveShell.editor') - ), - (('--log','-l'), dict( - action='store_true', dest='InteractiveShell.logstart', - help="Start logging to the default log file (./ipython_log.py).") - ), - (('--logfile','-lf'), dict( - type=unicode, dest='InteractiveShell.logfile', - help="Start logging to logfile with this name.", - metavar='InteractiveShell.logfile') - ), - (('--log-append','-la'), dict( - type=unicode, dest='InteractiveShell.logappend', - help="Start logging to the given file in append mode.", - metavar='InteractiveShell.logfile') - ), - (('--pdb',), dict( - action='store_true', dest='InteractiveShell.pdb', - help="Enable auto calling the pdb debugger after every exception.") - ), - (('--no-pdb',), dict( - action='store_false', dest='InteractiveShell.pdb', - help="Disable auto calling the pdb debugger after every exception.") - ), - (('--pprint',), dict( - action='store_true', dest='InteractiveShell.pprint', - help="Enable auto pretty printing of results.") - ), - (('--no-pprint',), dict( - action='store_false', dest='InteractiveShell.pprint', - help="Disable auto auto pretty printing of results.") - ), - (('--prompt-in1','-pi1'), dict( - type=str, dest='InteractiveShell.prompt_in1', - help= - """Set the main input prompt ('In [\#]: '). Note that if you are using - numbered prompts, the number is represented with a '\#' in the string. - Don't forget to quote strings with spaces embedded in them. Most - bash-like escapes can be used to customize IPython's prompts, as well - as a few additional ones which are IPython-spe- cific. All valid - prompt escapes are described in detail in the Customization section of - the IPython manual.""", - metavar='InteractiveShell.prompt_in1') - ), - (('--prompt-in2','-pi2'), dict( - type=str, dest='InteractiveShell.prompt_in2', - help= - """Set the secondary input prompt (' .\D.: '). Similar to the previous - option, but used for the continuation prompts. The special sequence - '\D' is similar to '\#', but with all digits replaced by dots (so you - can have your continuation prompt aligned with your input prompt). - Default: ' .\D.: ' (note three spaces at the start for alignment with - 'In [\#]')""", - metavar='InteractiveShell.prompt_in2') - ), - (('--prompt-out','-po'), dict( - type=str, dest='InteractiveShell.prompt_out', - help="Set the output prompt ('Out[\#]:')", - metavar='InteractiveShell.prompt_out') - ), - (('--quick',), dict( - action='store_true', dest='Global.quick', - help="Enable quick startup with no config files.") - ), - (('--readline',), dict( - action='store_true', dest='InteractiveShell.readline_use', - help="Enable readline for command line usage.") - ), - (('--no-readline',), dict( - action='store_false', dest='InteractiveShell.readline_use', - help="Disable readline for command line usage.") - ), - (('--screen-length','-sl'), dict( - type=int, dest='InteractiveShell.screen_length', - help= - """Number of lines of your screen, used to control printing of very - long strings. Strings longer than this number of lines will be sent - through a pager instead of directly printed. The default value for - this is 0, which means IPython will auto-detect your screen size every - time it needs to print certain potentially long strings (this doesn't - change the behavior of the 'print' keyword, it's only triggered - internally). If for some reason this isn't working well (it needs - curses support), specify it yourself. Otherwise don't change the - default.""", - metavar='InteractiveShell.screen_length') - ), - (('--separate-in','-si'), dict( - type=str, dest='InteractiveShell.separate_in', - help="Separator before input prompts. Default '\\n'.", - metavar='InteractiveShell.separate_in') - ), - (('--separate-out','-so'), dict( - type=str, dest='InteractiveShell.separate_out', - help="Separator before output prompts. Default 0 (nothing).", - metavar='InteractiveShell.separate_out') - ), - (('--separate-out2','-so2'), dict( - type=str, dest='InteractiveShell.separate_out2', - help="Separator after output prompts. Default 0 (nonight).", - metavar='InteractiveShell.separate_out2') - ), - (('-no-sep',), dict( - action='store_true', dest='Global.nosep', - help="Eliminate all spacing between prompts.") - ), - (('--term-title',), dict( - action='store_true', dest='InteractiveShell.term_title', - help="Enable auto setting the terminal title.") - ), - (('--no-term-title',), dict( - action='store_false', dest='InteractiveShell.term_title', - help="Disable auto setting the terminal title.") - ), - (('--xmode',), dict( - type=str, dest='InteractiveShell.xmode', - help= - """Exception reporting mode ('Plain','Context','Verbose'). Plain: - similar to python's normal traceback printing. Context: prints 5 lines - of context source code around each line in the traceback. Verbose: - similar to Context, but additionally prints the variables currently - visible where the exception happened (shortening their strings if too - long). This can potentially be very slow, if you happen to have a huge - data structure whose string representation is complex to compute. - Your computer may appear to freeze for a while with cpu usage at 100%%. - If this occurs, you can cancel the traceback with Ctrl-C (maybe hitting - it more than once). - """, - metavar='InteractiveShell.xmode') - ), - (('--ext',), dict( - type=str, dest='Global.extra_extension', - help="The dotted module name of an IPython extension to load.", - metavar='Global.extra_extension') - ), - (('-c',), dict( - type=str, dest='Global.code_to_run', - help="Execute the given command string.", - metavar='Global.code_to_run') - ), - (('-i',), dict( - action='store_true', dest='Global.force_interact', - help= - "If running code from the command line, become interactive afterwards." - ) - ), - - # Options to start with GUI control enabled from the beginning - (('--gui',), dict( - type=str, dest='Global.gui', - help="Enable GUI event loop integration ('qt', 'wx', 'gtk').", - metavar='gui-mode') - ), - - (('--pylab','-pylab'), dict( - type=str, dest='Global.pylab', - nargs='?', const='auto', metavar='gui-mode', - help="Pre-load matplotlib and numpy for interactive use. "+ - "If no value is given, the gui backend is matplotlib's, else use "+ - "one of: ['tk', 'qt', 'wx', 'gtk'].") - ), - - # Legacy GUI options. Leave them in for backwards compatibility, but the - # 'thread' names are really a misnomer now. - (('--wthread','-wthread'), dict( - action='store_true', dest='Global.wthread', - help="Enable wxPython event loop integration "+ - "(DEPRECATED, use --gui wx)") - ), - (('--q4thread','--qthread','-q4thread','-qthread'), dict( - action='store_true', dest='Global.q4thread', - help="Enable Qt4 event loop integration. Qt3 is no longer supported. "+ - "(DEPRECATED, use --gui qt)") - ), - (('--gthread','-gthread'), dict( - action='store_true', dest='Global.gthread', - help="Enable GTK event loop integration. "+ - "(DEPRECATED, use --gui gtk)") - ), -) + +class IPAppConfigLoader(BaseAppConfigLoader): + + def _add_arguments(self): + super(IPAppConfigLoader, self)._add_arguments() + paa = self.parser.add_argument + paa('-p', + '--profile', dest='Global.profile', type=unicode, + help= + """The string name of the ipython profile to be used. Assume that your + config file is ipython_config-.py (looks in current dir first, + then in IPYTHON_DIR). This is a quick way to keep and load multiple + config files for different tasks, especially if include your basic one + in your more specialized ones. You can keep a basic + IPYTHON_DIR/ipython_config.py file and then have other 'profiles' which + include this one and load extra things for particular tasks.""", + metavar='Global.profile') + paa('--config-file', + dest='Global.config_file', type=unicode, + help= + """Set the config file name to override default. Normally IPython + loads ipython_config.py (from current directory) or + IPYTHON_DIR/ipython_config.py. If the loading of your config file + fails, IPython starts with a bare bones configuration (no modules + loaded at all).""", + metavar='Global.config_file') + paa('--autocall', + dest='InteractiveShell.autocall', type=int, + help= + """Make IPython automatically call any callable object even if you + didn't type explicit parentheses. For example, 'str 43' becomes + 'str(43)' automatically. The value can be '0' to disable the feature, + '1' for 'smart' autocall, where it is not applied if there are no more + arguments on the line, and '2' for 'full' autocall, where all callable + objects are automatically called (even if no arguments are present). + The default is '1'.""", + metavar='InteractiveShell.autocall') + paa('--autoindent', + action='store_true', dest='InteractiveShell.autoindent', + help='Turn on autoindenting.') + paa('--no-autoindent', + action='store_false', dest='InteractiveShell.autoindent', + help='Turn off autoindenting.') + paa('--automagic', + action='store_true', dest='InteractiveShell.automagic', + help= + """Turn on the auto calling of magic commands. Type %%magic at the + IPython prompt for more information.""") + paa('--no-automagic', + action='store_false', dest='InteractiveShell.automagic', + help='Turn off the auto calling of magic commands.') + paa('--autoedit-syntax', + action='store_true', dest='InteractiveShell.autoedit_syntax', + help='Turn on auto editing of files with syntax errors.') + paa('--no-autoedit-syntax', + action='store_false', dest='InteractiveShell.autoedit_syntax', + help='Turn off auto editing of files with syntax errors.') + paa('--banner', + action='store_true', dest='Global.display_banner', + help='Display a banner upon starting IPython.') + paa('--no-banner', + action='store_false', dest='Global.display_banner', + help="Don't display a banner upon starting IPython.") + paa('--cache-size', + type=int, dest='InteractiveShell.cache_size', + help= + """Set the size of the output cache. The default is 1000, you can + change it permanently in your config file. Setting it to 0 completely + disables the caching system, and the minimum value accepted is 20 (if + you provide a value less than 20, it is reset to 0 and a warning is + issued). This limit is defined because otherwise you'll spend more + time re-flushing a too small cache than working""", + metavar='InteractiveShell.cache_size') + paa('--classic', + action='store_true', dest='Global.classic', + help="Gives IPython a similar feel to the classic Python prompt.") + paa('--colors', + type=str, dest='InteractiveShell.colors', + help="Set the color scheme (NoColor, Linux, and LightBG).", + metavar='InteractiveShell.colors') + paa('--color-info', + action='store_true', dest='InteractiveShell.color_info', + help= + """IPython can display information about objects via a set of func- + tions, and optionally can use colors for this, syntax highlighting + source code and various other elements. However, because this + information is passed through a pager (like 'less') and many pagers get + confused with color codes, this option is off by default. You can test + it and turn it on permanently in your ipython_config.py file if it + works for you. Test it and turn it on permanently if it works with + your system. The magic function %%color_info allows you to toggle this + inter- actively for testing.""") + paa('--no-color-info', + action='store_false', dest='InteractiveShell.color_info', + help="Disable using colors for info related things.") + paa('--confirm-exit', + action='store_true', dest='InteractiveShell.confirm_exit', + help= + """Set to confirm when you try to exit IPython with an EOF (Control-D + in Unix, Control-Z/Enter in Windows). By typing 'exit', 'quit' or + '%%Exit', you can force a direct exit without any confirmation.""") + paa('--no-confirm-exit', + action='store_false', dest='InteractiveShell.confirm_exit', + help="Don't prompt the user when exiting.") + paa('--deep-reload', + action='store_true', dest='InteractiveShell.deep_reload', + help= + """Enable deep (recursive) reloading by default. IPython can use the + deep_reload module which reloads changes in modules recursively (it + replaces the reload() function, so you don't need to change anything to + use it). deep_reload() forces a full reload of modules whose code may + have changed, which the default reload() function does not. When + deep_reload is off, IPython will use the normal reload(), but + deep_reload will still be available as dreload(). This fea- ture is off + by default [which means that you have both normal reload() and + dreload()].""") + paa('--no-deep-reload', + action='store_false', dest='InteractiveShell.deep_reload', + help="Disable deep (recursive) reloading by default.") + paa('--editor', + type=str, dest='InteractiveShell.editor', + help="Set the editor used by IPython (default to $EDITOR/vi/notepad).", + metavar='InteractiveShell.editor') + paa('--log','-l', + action='store_true', dest='InteractiveShell.logstart', + help="Start logging to the default log file (./ipython_log.py).") + paa('--logfile','-lf', + type=unicode, dest='InteractiveShell.logfile', + help="Start logging to logfile with this name.", + metavar='InteractiveShell.logfile') + paa('--log-append','-la', + type=unicode, dest='InteractiveShell.logappend', + help="Start logging to the given file in append mode.", + metavar='InteractiveShell.logfile') + paa('--pdb', + action='store_true', dest='InteractiveShell.pdb', + help="Enable auto calling the pdb debugger after every exception.") + paa('--no-pdb', + action='store_false', dest='InteractiveShell.pdb', + help="Disable auto calling the pdb debugger after every exception.") + paa('--pprint', + action='store_true', dest='InteractiveShell.pprint', + help="Enable auto pretty printing of results.") + paa('--no-pprint', + action='store_false', dest='InteractiveShell.pprint', + help="Disable auto auto pretty printing of results.") + paa('--prompt-in1','-pi1', + type=str, dest='InteractiveShell.prompt_in1', + help= + """Set the main input prompt ('In [\#]: '). Note that if you are using + numbered prompts, the number is represented with a '\#' in the string. + Don't forget to quote strings with spaces embedded in them. Most + bash-like escapes can be used to customize IPython's prompts, as well + as a few additional ones which are IPython-spe- cific. All valid + prompt escapes are described in detail in the Customization section of + the IPython manual.""", + metavar='InteractiveShell.prompt_in1') + paa('--prompt-in2','-pi2', + type=str, dest='InteractiveShell.prompt_in2', + help= + """Set the secondary input prompt (' .\D.: '). Similar to the previous + option, but used for the continuation prompts. The special sequence + '\D' is similar to '\#', but with all digits replaced by dots (so you + can have your continuation prompt aligned with your input prompt). + Default: ' .\D.: ' (note three spaces at the start for alignment with + 'In [\#]')""", + metavar='InteractiveShell.prompt_in2') + paa('--prompt-out','-po', + type=str, dest='InteractiveShell.prompt_out', + help="Set the output prompt ('Out[\#]:')", + metavar='InteractiveShell.prompt_out') + paa('--quick', + action='store_true', dest='Global.quick', + help="Enable quick startup with no config files.") + paa('--readline', + action='store_true', dest='InteractiveShell.readline_use', + help="Enable readline for command line usage.") + paa('--no-readline', + action='store_false', dest='InteractiveShell.readline_use', + help="Disable readline for command line usage.") + paa('--screen-length','-sl', + type=int, dest='InteractiveShell.screen_length', + help= + """Number of lines of your screen, used to control printing of very + long strings. Strings longer than this number of lines will be sent + through a pager instead of directly printed. The default value for + this is 0, which means IPython will auto-detect your screen size every + time it needs to print certain potentially long strings (this doesn't + change the behavior of the 'print' keyword, it's only triggered + internally). If for some reason this isn't working well (it needs + curses support), specify it yourself. Otherwise don't change the + default.""", + metavar='InteractiveShell.screen_length') + paa('--separate-in','-si', + type=str, dest='InteractiveShell.separate_in', + help="Separator before input prompts. Default '\\n'.", + metavar='InteractiveShell.separate_in') + paa('--separate-out','-so', + type=str, dest='InteractiveShell.separate_out', + help="Separator before output prompts. Default 0 (nothing).", + metavar='InteractiveShell.separate_out') + paa('--separate-out2','-so2', + type=str, dest='InteractiveShell.separate_out2', + help="Separator after output prompts. Default 0 (nonight).", + metavar='InteractiveShell.separate_out2') + paa('--no-sep', + action='store_true', dest='Global.nosep', + help="Eliminate all spacing between prompts.") + paa('--term-title', + action='store_true', dest='InteractiveShell.term_title', + help="Enable auto setting the terminal title.") + paa('--no-term-title', + action='store_false', dest='InteractiveShell.term_title', + help="Disable auto setting the terminal title.") + paa('--xmode', + type=str, dest='InteractiveShell.xmode', + help= + """Exception reporting mode ('Plain','Context','Verbose'). Plain: + similar to python's normal traceback printing. Context: prints 5 lines + of context source code around each line in the traceback. Verbose: + similar to Context, but additionally prints the variables currently + visible where the exception happened (shortening their strings if too + long). This can potentially be very slow, if you happen to have a huge + data structure whose string representation is complex to compute. + Your computer may appear to freeze for a while with cpu usage at 100%%. + If this occurs, you can cancel the traceback with Ctrl-C (maybe hitting + it more than once). + """, + metavar='InteractiveShell.xmode') + paa('--ext', + type=str, dest='Global.extra_extension', + help="The dotted module name of an IPython extension to load.", + metavar='Global.extra_extension') + paa('-c', + type=str, dest='Global.code_to_run', + help="Execute the given command string.", + metavar='Global.code_to_run') + paa('-i', + action='store_true', dest='Global.force_interact', + help= + "If running code from the command line, become interactive afterwards.") + + # Options to start with GUI control enabled from the beginning + paa('--gui', + type=str, dest='Global.gui', + help="Enable GUI event loop integration ('qt', 'wx', 'gtk').", + metavar='gui-mode') + paa('--pylab','-pylab', + type=str, dest='Global.pylab', + nargs='?', const='auto', metavar='gui-mode', + help="Pre-load matplotlib and numpy for interactive use. "+ + "If no value is given, the gui backend is matplotlib's, else use "+ + "one of: ['tk', 'qt', 'wx', 'gtk'].") + + # Legacy GUI options. Leave them in for backwards compatibility, but the + # 'thread' names are really a misnomer now. + paa('--wthread', '-wthread', + action='store_true', dest='Global.wthread', + help= + """Enable wxPython event loop integration. (DEPRECATED, use --gui wx)""") + paa('--q4thread', '--qthread', '-q4thread', '-qthread', + action='store_true', dest='Global.q4thread', + help= + """Enable Qt4 event loop integration. Qt3 is no longer supported. + (DEPRECATED, use --gui qt)""") + paa('--gthread', '-gthread', + action='store_true', dest='Global.gthread', + help= + """Enable GTK event loop integration. (DEPRECATED, use --gui gtk)""") + #----------------------------------------------------------------------------- -# Main classes and functions +# Crash handler for this application #----------------------------------------------------------------------------- -class IPythonApp(Application): - name = u'ipython' - #: argparse formats better the 'usage' than the 'description' field - description = None - #: usage message printed by argparse. If None, auto-generate - usage = usage.cl_usage - config_file_name = default_config_file_name +_message_template = """\ +Oops, $self.app_name crashed. We do our best to make it stable, but... + +A crash report was automatically generated with the following information: + - A verbatim copy of the crash traceback. + - A copy of your input history during this session. + - Data on your current $self.app_name configuration. - cl_arguments = Application.cl_arguments + cl_args +It was left in the file named: +\t'$self.crash_report_fname' +If you can email this file to the developers, the information in it will help +them in understanding and correcting the problem. - # Private and configuration attributes - _CrashHandler = crashhandler.IPythonCrashHandler - - def __init__(self, argv=None, - constructor_config=None, override_config=None, - **shell_params): - """Create a new IPythonApp. +You can mail it to: $self.contact_name at $self.contact_email +with the subject '$self.app_name Crash Report'. - See the parent class for details on how configuration is handled. +If you want to do it now, the following command will work (under Unix): +mail -s '$self.app_name Crash Report' $self.contact_email < $self.crash_report_fname - Parameters - ---------- - argv : optional, list - If given, used as the command-line argv environment to read arguments - from. +To ensure accurate tracking of this issue, please file a report about it at: +$self.bug_tracker +""" - constructor_config : optional, Config - If given, additional config that is merged last, after internal - defaults, command-line and file-based configs. +class IPAppCrashHandler(CrashHandler): + """sys.excepthook for IPython itself, leaves a detailed report on disk.""" - override_config : optional, Config - If given, config that overrides all others unconditionally (except - for internal defaults, which ensure that all parameters exist). + message_template = _message_template - shell_params : optional, dict - All other keywords are passed to the :class:`iplib.InteractiveShell` - constructor. - """ - super(IPythonApp, self).__init__(argv, constructor_config, - override_config) - self.shell_params = shell_params + def __init__(self, app): + contact_name = release.authors['Fernando'][0] + contact_email = release.authors['Fernando'][1] + bug_tracker = 'https://bugs.launchpad.net/ipython/+filebug' + super(IPAppCrashHandler,self).__init__( + app, contact_name, contact_email, bug_tracker + ) + + def make_report(self,traceback): + """Return a string containing a crash report.""" + + sec_sep = self.section_sep + # Start with parent report + report = [super(IPAppCrashHandler, self).make_report(traceback)] + # Add interactive-specific info we may have + rpt_add = report.append + try: + rpt_add(sec_sep+"History of session input:") + for line in self.app.shell.user_ns['_ih']: + rpt_add(line) + rpt_add('\n*** Last line of input (may not be in above history):\n') + rpt_add(self.app.shell._last_input_line+'\n') + except: + pass + + return ''.join(report) + + +#----------------------------------------------------------------------------- +# Main classes and functions +#----------------------------------------------------------------------------- + +class IPythonApp(Application): + name = u'ipython' + #: argparse formats better the 'usage' than the 'description' field + description = None + usage = usage.cl_usage + command_line_loader = IPAppConfigLoader + default_config_file_name = default_config_file_name + crash_handler_class = IPAppCrashHandler def create_default_config(self): super(IPythonApp, self).create_default_config() @@ -474,8 +476,7 @@ class IPythonApp(Application): sys.path.insert(0, '') # Create an InteractiveShell instance - self.shell = InteractiveShell(None, self.master_config, - **self.shell_params ) + self.shell = InteractiveShell(None, self.master_config) def post_construct(self): """Do actions after construct, but before starting the app.""" @@ -485,7 +486,6 @@ class IPythonApp(Application): # based app, because we call shell.show_banner() by hand below # so the banner shows *before* all extension loading stuff. self.shell.display_banner = False - if config.Global.display_banner and \ config.Global.interact: self.shell.show_banner() @@ -499,7 +499,6 @@ class IPythonApp(Application): self._run_exec_lines() self._run_exec_files() self._run_cmd_line_code() - self._configure_xmode() def _enable_gui_pylab(self): """Enable GUI event loop integration, taking pylab into account.""" @@ -624,11 +623,6 @@ class IPythonApp(Application): self.log.warn("Error in executing file in user namespace: %s" % fname) self.shell.showtraceback() - def _configure_xmode(self): - # XXX - shouldn't this be read from the config? I'm still a little - # lost with all the details of handling the new config guys... - self.shell.InteractiveTB.set_mode(mode=self.shell.xmode) - def start_app(self): if self.master_config.Global.interact: self.log.debug("Starting IPython's mainloop...") @@ -653,3 +647,7 @@ def launch_new_instance(): """Create and run a full blown IPython instance""" app = IPythonApp() app.start() + + +if __name__ == '__main__': + launch_new_instance() diff --git a/IPython/core/iplib.py b/IPython/core/iplib.py index 7fc2f8b..c75b404 100644 --- a/IPython/core/iplib.py +++ b/IPython/core/iplib.py @@ -20,7 +20,6 @@ from __future__ import with_statement from __future__ import absolute_import import __builtin__ -import StringIO import bdb import codeop import exceptions @@ -47,29 +46,35 @@ from IPython.core.logger import Logger from IPython.core.magic import Magic from IPython.core.prefilter import PrefilterManager from IPython.core.prompts import CachedOutput -from IPython.core.pylabtools import pylab_activate from IPython.core.usage import interactive_usage, default_banner +import IPython.core.hooks from IPython.external.Itpl import ItplNS from IPython.lib.inputhook import enable_gui from IPython.lib.backgroundjobs import BackgroundJobManager +from IPython.lib.pylabtools import pylab_activate from IPython.utils import PyColorize from IPython.utils import pickleshare -from IPython.utils.genutils import get_ipython_dir +from IPython.utils.doctestreload import doctest_reload from IPython.utils.ipstruct import Struct -from IPython.utils.platutils import toggle_set_term_title, set_term_title +from IPython.utils.io import Term, ask_yes_no +from IPython.utils.path import get_home_dir, get_ipython_dir, HomeDirError +from IPython.utils.process import ( + abbrev_cwd, + getoutput, + getoutputerror +) +# import IPython.utils.rlineimpl as readline from IPython.utils.strdispatch import StrDispatch from IPython.utils.syspathcontext import prepended_to_syspath - -# XXX - need to clean up this import * line -from IPython.utils.genutils import * - -# from IPython.utils import growl -# growl.start("IPython") - +from IPython.utils.terminal import toggle_set_term_title, set_term_title +from IPython.utils.warn import warn, error, fatal from IPython.utils.traitlets import ( Int, Str, CBool, CaselessStrEnum, Enum, List, Unicode ) +# from IPython.utils import growl +# growl.start("IPython") + #----------------------------------------------------------------------------- # Globals #----------------------------------------------------------------------------- @@ -186,64 +191,6 @@ class SeparateStr(Str): return super(SeparateStr, self).validate(obj, value) -def make_user_namespaces(user_ns=None, user_global_ns=None): - """Return a valid local and global user interactive namespaces. - - This builds a dict with the minimal information needed to operate as a - valid IPython user namespace, which you can pass to the various - embedding classes in ipython. The default implementation returns the - same dict for both the locals and the globals to allow functions to - refer to variables in the namespace. Customized implementations can - return different dicts. The locals dictionary can actually be anything - following the basic mapping protocol of a dict, but the globals dict - must be a true dict, not even a subclass. It is recommended that any - custom object for the locals namespace synchronize with the globals - dict somehow. - - Raises TypeError if the provided globals namespace is not a true dict. - - Parameters - ---------- - user_ns : dict-like, optional - The current user namespace. The items in this namespace should - be included in the output. If None, an appropriate blank - namespace should be created. - user_global_ns : dict, optional - The current user global namespace. The items in this namespace - should be included in the output. If None, an appropriate - blank namespace should be created. - - Returns - ------- - A pair of dictionary-like object to be used as the local namespace - of the interpreter and a dict to be used as the global namespace. - """ - - - # We must ensure that __builtin__ (without the final 's') is always - # available and pointing to the __builtin__ *module*. For more details: - # http://mail.python.org/pipermail/python-dev/2001-April/014068.html - - if user_ns is None: - # Set __name__ to __main__ to better match the behavior of the - # normal interpreter. - user_ns = {'__name__' :'__main__', - '__builtin__' : __builtin__, - '__builtins__' : __builtin__, - } - else: - user_ns.setdefault('__name__','__main__') - user_ns.setdefault('__builtin__',__builtin__) - user_ns.setdefault('__builtins__',__builtin__) - - if user_global_ns is None: - user_global_ns = user_ns - if type(user_global_ns) is not dict: - raise TypeError("user_global_ns must be a true dict; got %r" - % type(user_global_ns)) - - return user_ns, user_global_ns - #----------------------------------------------------------------------------- # Main IPython class #----------------------------------------------------------------------------- @@ -658,7 +605,6 @@ class InteractiveShell(Component, Magic): self.strdispatchers = {} # Set all default hooks, defined in the IPython.hooks module. - import IPython.core.hooks hooks = IPython.core.hooks for hook_name in hooks.__all__: # default hooks have priority 100, i.e. low; user hooks should have @@ -876,7 +822,7 @@ class InteractiveShell(Component, Magic): # These routines return properly built dicts as needed by the rest of # the code, and can also be used by extension writers to generate # properly initialized namespaces. - user_ns, user_global_ns = make_user_namespaces(user_ns, user_global_ns) + user_ns, user_global_ns = self.make_user_namespaces(user_ns, user_global_ns) # Assign namespaces # This is the namespace where all normal user variables live @@ -887,7 +833,7 @@ class InteractiveShell(Component, Magic): # loaded at startup, so we can list later only variables defined in # actual interactive use. Since it is always a subset of user_ns, it # doesn't need to be separately tracked in the ns_table. - self.user_config_ns = {} + self.user_ns_hidden = {} # A namespace to keep track of internal data structures to prevent # them from cluttering user-visible stuff. Will be updated later @@ -933,9 +879,67 @@ class InteractiveShell(Component, Magic): # Similarly, track all namespaces where references can be held and that # we can safely clear (so it can NOT include builtin). This one can be # a simple list. - self.ns_refs_table = [ user_ns, user_global_ns, self.user_config_ns, + self.ns_refs_table = [ user_ns, user_global_ns, self.user_ns_hidden, self.internal_ns, self._main_ns_cache ] + def make_user_namespaces(self, user_ns=None, user_global_ns=None): + """Return a valid local and global user interactive namespaces. + + This builds a dict with the minimal information needed to operate as a + valid IPython user namespace, which you can pass to the various + embedding classes in ipython. The default implementation returns the + same dict for both the locals and the globals to allow functions to + refer to variables in the namespace. Customized implementations can + return different dicts. The locals dictionary can actually be anything + following the basic mapping protocol of a dict, but the globals dict + must be a true dict, not even a subclass. It is recommended that any + custom object for the locals namespace synchronize with the globals + dict somehow. + + Raises TypeError if the provided globals namespace is not a true dict. + + Parameters + ---------- + user_ns : dict-like, optional + The current user namespace. The items in this namespace should + be included in the output. If None, an appropriate blank + namespace should be created. + user_global_ns : dict, optional + The current user global namespace. The items in this namespace + should be included in the output. If None, an appropriate + blank namespace should be created. + + Returns + ------- + A pair of dictionary-like object to be used as the local namespace + of the interpreter and a dict to be used as the global namespace. + """ + + + # We must ensure that __builtin__ (without the final 's') is always + # available and pointing to the __builtin__ *module*. For more details: + # http://mail.python.org/pipermail/python-dev/2001-April/014068.html + + if user_ns is None: + # Set __name__ to __main__ to better match the behavior of the + # normal interpreter. + user_ns = {'__name__' :'__main__', + '__builtin__' : __builtin__, + '__builtins__' : __builtin__, + } + else: + user_ns.setdefault('__name__','__main__') + user_ns.setdefault('__builtin__',__builtin__) + user_ns.setdefault('__builtins__',__builtin__) + + if user_global_ns is None: + user_global_ns = user_ns + if type(user_global_ns) is not dict: + raise TypeError("user_global_ns must be a true dict; got %r" + % type(user_global_ns)) + + return user_ns, user_global_ns + def init_sys_modules(self): # We need to insert into sys.modules something that looks like a # module but which accesses the IPython namespace, for shelve and @@ -974,7 +978,7 @@ class InteractiveShell(Component, Magic): therm. """ # This function works in two parts: first we put a few things in - # user_ns, and we sync that contents into user_config_ns so that these + # user_ns, and we sync that contents into user_ns_hidden so that these # initial variables aren't shown by %who. After the sync, we add the # rest of what we *do* want the user to see with %who even on a new # session (probably nothing, so theye really only see their own stuff) @@ -1014,9 +1018,9 @@ class InteractiveShell(Component, Magic): # Store myself as the public api!!! ns['get_ipython'] = self.get_ipython - # Sync what we've added so far to user_config_ns so these aren't seen + # Sync what we've added so far to user_ns_hidden so these aren't seen # by %who - self.user_config_ns.update(ns) + self.user_ns_hidden.update(ns) # Anything put into ns now would show up in %who. Think twice before # putting anything here, as we really want %who to show the user their @@ -1089,7 +1093,7 @@ class InteractiveShell(Component, Magic): self.user_ns.update(vdict) # And configure interactive visibility - config_ns = self.user_config_ns + config_ns = self.user_ns_hidden if interactive: for name, val in vdict.iteritems(): config_ns.pop(name, None) @@ -1164,7 +1168,9 @@ class InteractiveShell(Component, Magic): Convert func into callable that saves & restores history around the call """ - if not self.has_readline: + if self.has_readline: + from IPython.utils import rlineimpl as readline + else: return func def wrapper(): @@ -1198,6 +1204,9 @@ class InteractiveShell(Component, Magic): # and add any custom exception handlers the user may have specified self.set_custom_exc(*custom_exceptions) + # Set the exception mode + self.InteractiveTB.set_mode(mode=self.xmode) + def set_custom_exc(self,exc_tuple,handler): """set_custom_exc(exc_tuple,handler) @@ -2351,6 +2360,9 @@ class InteractiveShell(Component, Magic): to make it easy to write extensions, you can also put your extensions in ``os.path.join(self.ipython_dir, 'extensions')``. This directory is added to ``sys.path`` automatically. + + If :func:`load_ipython_extension` returns anything, this function + will return that object. """ from IPython.utils.syspathcontext import prepended_to_syspath @@ -2495,11 +2507,11 @@ class InteractiveShell(Component, Magic): # We want to prevent the loading of pylab to pollute the user's # namespace as shown by the %who* magics, so we execute the activation # code in an empty namespace, and we update *both* user_ns and - # user_config_ns with this information. + # user_ns_hidden with this information. ns = {} gui = pylab_activate(ns, gui) self.user_ns.update(ns) - self.user_config_ns.update(ns) + self.user_ns_hidden.update(ns) # Now we must activate the gui pylab wants to use, and fix %run to take # plot updates into account enable_gui(gui) diff --git a/IPython/core/macro.py b/IPython/core/macro.py index 7ca39ed..8fb52a6 100644 --- a/IPython/core/macro.py +++ b/IPython/core/macro.py @@ -7,7 +7,7 @@ # the file COPYING, distributed as part of this software. #***************************************************************************** -from IPython.utils.genutils import Term +from IPython.utils.io import Term from IPython.core.autocall import IPyAutocall class Macro(IPyAutocall): diff --git a/IPython/core/magic.py b/IPython/core/magic.py index 615f545..85b5a79 100644 --- a/IPython/core/magic.py +++ b/IPython/core/magic.py @@ -1,35 +1,33 @@ -# -*- coding: utf-8 -*- +# encoding: utf-8 """Magic functions for InteractiveShell. """ -#***************************************************************************** -# Copyright (C) 2001 Janko Hauser and -# Copyright (C) 2001-2006 Fernando Perez -# +#----------------------------------------------------------------------------- +# Copyright (C) 2001 Janko Hauser and +# Copyright (C) 2001-2007 Fernando Perez +# Copyright (C) 2008-2009 The IPython Development Team + # Distributed under the terms of the BSD License. The full license is in # the file COPYING, distributed as part of this software. -#***************************************************************************** +#----------------------------------------------------------------------------- -#**************************************************************************** -# Modules and globals +#----------------------------------------------------------------------------- +# Imports +#----------------------------------------------------------------------------- -# Python standard modules import __builtin__ import bdb import inspect import os -import pdb -import pydoc import sys import shutil import re -import tempfile import time -import cPickle as pickle import textwrap +import types from cStringIO import StringIO from getopt import getopt,GetoptError -from pprint import pprint, pformat +from pprint import pformat # cProfile was added in Python2.5 try: @@ -42,10 +40,7 @@ except ImportError: except ImportError: profile = pstats = None -# Homebrewed import IPython -import IPython.utils.generics - from IPython.core import debugger, oinspect from IPython.core.error import TryNext from IPython.core.error import UsageError @@ -53,20 +48,24 @@ from IPython.core.fakemodule import FakeModule from IPython.core.macro import Macro from IPython.core.page import page from IPython.core.prefilter import ESC_MAGIC -from IPython.core.pylabtools import mpl_runner +from IPython.lib.pylabtools import mpl_runner from IPython.lib.inputhook import enable_gui -from IPython.external.Itpl import Itpl, itpl, printpl,itplns +from IPython.external.Itpl import itpl, printpl from IPython.testing import decorators as testdec -from IPython.utils import platutils -from IPython.utils import wildcard -from IPython.utils.PyColorize import Parser +from IPython.utils.io import Term, file_read, nlprint +from IPython.utils.path import get_py_filename +from IPython.utils.process import arg_split, abbrev_cwd +from IPython.utils.terminal import set_term_title +from IPython.utils.text import LSString, SList, StringTypes +from IPython.utils.timing import clock, clock2 +from IPython.utils.warn import warn, error from IPython.utils.ipstruct import Struct +import IPython.utils.generics -# XXX - We need to switch to explicit imports here with genutils -from IPython.utils.genutils import * - -#*************************************************************************** +#----------------------------------------------------------------------------- # Utility functions +#----------------------------------------------------------------------------- + def on_off(tag): """Return an ON/OFF string for a 1/0 input. Simple utility function.""" return ['OFF','ON'][tag] @@ -94,6 +93,9 @@ def compress_dhist(dh): # on construction of the main InteractiveShell object. Something odd is going # on with super() calls, Component and the MRO... For now leave it as-is, but # eventually this needs to be clarified. +# BG: This is because InteractiveShell inherits from this, but is itself a +# Component. This messes up the MRO in some way. The fix is that we need to +# make Magic a component that InteractiveShell does not subclass. class Magic: """Magic functions for InteractiveShell. @@ -277,7 +279,7 @@ python-profiler package from non-free.""") def arg_err(self,func): """Print docstring if incorrect arguments were passed""" print 'Error in arguments:' - print OInspect.getdoc(func) + print oinspect.getdoc(func) def format_latex(self,strng): """Format a string for latex inclusion.""" @@ -884,10 +886,10 @@ Currently the magic system has the following functions:\n""" user_ns = self.shell.user_ns internal_ns = self.shell.internal_ns - user_config_ns = self.shell.user_config_ns + user_ns_hidden = self.shell.user_ns_hidden out = [ i for i in user_ns if not i.startswith('_') \ - and not (i in internal_ns or i in user_config_ns) ] + and not (i in internal_ns or i in user_ns_hidden) ] typelist = parameter_s.split() if typelist: @@ -1170,7 +1172,7 @@ Currently the magic system has the following functions:\n""" started = logger.logstart(logfname,loghead,logmode, log_output,timestamp,log_raw_input) except: - rc.opts.logfile = old_logfile + self.shell.logfile = old_logfile warn("Couldn't start log: %s" % sys.exc_info()[1]) else: # log input history up to this point, optionally interleaving @@ -2811,7 +2813,7 @@ Defaulting color scheme to 'NoColor'""" try: os.chdir(os.path.expanduser(ps)) if self.shell.term_title: - platutils.set_term_title('IPython: ' + abbrev_cwd()) + set_term_title('IPython: ' + abbrev_cwd()) except OSError: print sys.exc_info()[1] else: @@ -2824,7 +2826,7 @@ Defaulting color scheme to 'NoColor'""" else: os.chdir(self.shell.home_dir) if self.shell.term_title: - platutils.set_term_title('IPython: ' + '~') + set_term_title('IPython: ' + '~') cwd = os.getcwd() dhist = self.shell.user_ns['_dh'] diff --git a/IPython/core/oinspect.py b/IPython/core/oinspect.py index a0cfb54..91ac5a5 100644 --- a/IPython/core/oinspect.py +++ b/IPython/core/oinspect.py @@ -27,10 +27,11 @@ import sys import types # IPython's own -from IPython.utils import PyColorize -from IPython.utils.genutils import indent, Term from IPython.core.page import page from IPython.external.Itpl import itpl +from IPython.utils import PyColorize +from IPython.utils.io import Term +from IPython.utils.text import indent from IPython.utils.wildcard import list_namespace from IPython.utils.coloransi import * diff --git a/IPython/core/page.py b/IPython/core/page.py index f07c1b5..7226db7 100644 --- a/IPython/core/page.py +++ b/IPython/core/page.py @@ -30,15 +30,15 @@ rid of that dependency, we could move it there. import os import re import sys +import tempfile from IPython.core import ipapi from IPython.core.error import TryNext -from IPython.utils.genutils import ( - chop, Term, USE_CURSES -) - -if os.name == "nt": - from IPython.utils.winconsole import get_console_size +from IPython.utils.cursesimport import use_curses +from IPython.utils.data import chop +from IPython.utils.io import Term +from IPython.utils.process import xsys +from IPython.utils.terminal import get_terminal_size #----------------------------------------------------------------------------- @@ -47,7 +47,7 @@ if os.name == "nt": esc_re = re.compile(r"(\x1b[^m]+m)") -def page_dumb(strng,start=0,screen_lines=25): +def page_dumb(strng, start=0, screen_lines=25): """Very dumb 'pager' in Python, for when nothing else works. Only moves forward, same interface as page(), except for pager_cmd and @@ -69,8 +69,8 @@ def page_dumb(strng,start=0,screen_lines=25): last_escape = esc_list[-1] print >>Term.cout, last_escape + os.linesep.join(screens[-1]) -#---------------------------------------------------------------------------- -def page(strng,start=0,screen_lines=0,pager_cmd = None): + +def page(strng, start=0, screen_lines=0, pager_cmd=None): """Print a string, piping through a pager after a certain length. The screen_lines parameter specifies the number of *usable* lines of your @@ -93,7 +93,7 @@ def page(strng,start=0,screen_lines=0,pager_cmd = None): # Some routines may auto-compute start offsets incorrectly and pass a # negative value. Offset to 0 for robustness. - start = max(0,start) + start = max(0, start) # first, try the hook ip = ipapi.get() @@ -120,19 +120,16 @@ def page(strng,start=0,screen_lines=0,pager_cmd = None): # terminals. If someone later feels like refining it, it's not hard. numlines = max(num_newlines,int(len_str/80)+1) - if os.name == "nt": - screen_lines_def = get_console_size(defaulty=25)[1] - else: - screen_lines_def = 25 # default value if we can't auto-determine + screen_lines_def = get_terminal_size()[1] # auto-determine screen size if screen_lines <= 0: if TERM=='xterm' or TERM=='xterm-color': - use_curses = USE_CURSES + local_use_curses = use_curses else: # curses causes problems on many terminals other than xterm. - use_curses = False - if use_curses: + local_use_curses = False + if local_use_curses: import termios import curses # There is a bug in curses, where *sometimes* it fails to properly @@ -201,8 +198,8 @@ def page(strng,start=0,screen_lines=0,pager_cmd = None): if retval is not None: page_dumb(strng,screen_lines=screen_lines) -#---------------------------------------------------------------------------- -def page_file(fname,start = 0, pager_cmd = None): + +def page_file(fname, start=0, pager_cmd=None): """Page a file, using an optional pager command and starting line. """ @@ -221,12 +218,12 @@ def page_file(fname,start = 0, pager_cmd = None): except: print 'Unable to show file',`fname` -#---------------------------------------------------------------------------- -def get_pager_cmd(pager_cmd = None): - """Return a pager command. - Makes some attempts at finding an OS-correct one.""" +def get_pager_cmd(pager_cmd=None): + """Return a pager command. + Makes some attempts at finding an OS-correct one. + """ if os.name == 'posix': default_pager_cmd = 'less -r' # -r for color control sequences elif os.name in ['nt','dos']: @@ -239,8 +236,8 @@ def get_pager_cmd(pager_cmd = None): pager_cmd = default_pager_cmd return pager_cmd -#----------------------------------------------------------------------------- -def get_pager_start(pager,start): + +def get_pager_start(pager, start): """Return the string for paging files with an offset. This is the '+N' argument which less and more (under Unix) accept. @@ -255,8 +252,8 @@ def get_pager_start(pager,start): start_string = '' return start_string -#---------------------------------------------------------------------------- -# (X)emacs on W32 doesn't like to be bypassed with msvcrt.getch() + +# (X)emacs on win32 doesn't like to be bypassed with msvcrt.getch() if os.name == 'nt' and os.environ.get('TERM','dumb') != 'emacs': import msvcrt def page_more(): @@ -280,7 +277,7 @@ else: else: return True -#---------------------------------------------------------------------------- + def snip_print(str,width = 75,print_full = 0,header = ''): """Print a string snipping the midsection to fit in width. @@ -305,4 +302,5 @@ def snip_print(str,width = 75,print_full = 0,header = ''): if snip and print_full == 2: if raw_input(header+' Snipped. View (y/n)? [N]').lower() == 'y': page(str) - return snip \ No newline at end of file + return snip + diff --git a/IPython/core/prefilter.py b/IPython/core/prefilter.py index a7a4f10..d77fa69 100755 --- a/IPython/core/prefilter.py +++ b/IPython/core/prefilter.py @@ -27,10 +27,7 @@ Authors: import __builtin__ import codeop -import keyword -import os import re -import sys from IPython.core.alias import AliasManager from IPython.core.autocall import IPyAutocall @@ -39,7 +36,8 @@ from IPython.core.splitinput import split_user_input from IPython.core.page import page from IPython.utils.traitlets import List, Int, Any, Str, CBool, Bool -from IPython.utils.genutils import make_quoted_expr, Term +from IPython.utils.io import Term +from IPython.utils.text import make_quoted_expr from IPython.utils.autoattr import auto_attr #----------------------------------------------------------------------------- @@ -158,11 +156,12 @@ class LineInfo(object): without worrying about *further* damaging state. """ if not self._oinfo: + # ip.shell._ofind is actually on the Magic class! self._oinfo = ip.shell._ofind(self.ifun) return self._oinfo def __str__(self): - return "Lineinfo [%s|%s|%s]" %(self.pre,self.ifun,self.the_rest) + return "Lineinfo [%s|%s|%s]" %(self.pre, self.ifun, self.the_rest) #----------------------------------------------------------------------------- diff --git a/IPython/core/prompts.py b/IPython/core/prompts.py index 16aa422..2220fdb 100644 --- a/IPython/core/prompts.py +++ b/IPython/core/prompts.py @@ -12,23 +12,20 @@ Classes for handling input/output prompts. #***************************************************************************** #**************************************************************************** -# Required modules + import __builtin__ import os +import re import socket import sys -import time -# IPython's own -from IPython.utils import coloransi from IPython.core import release from IPython.external.Itpl import ItplNS from IPython.core.error import TryNext -from IPython.utils.ipstruct import Struct -from IPython.core.macro import Macro +from IPython.utils import coloransi import IPython.utils.generics - -from IPython.utils.genutils import * +from IPython.utils.warn import warn +from IPython.utils.io import Term #**************************************************************************** #Color schemes for Prompts. diff --git a/IPython/core/quitter.py b/IPython/core/quitter.py index 0bdaffb..19f7da0 100755 --- a/IPython/core/quitter.py +++ b/IPython/core/quitter.py @@ -19,7 +19,11 @@ Authors # Imports #----------------------------------------------------------------------------- -import sys + +#----------------------------------------------------------------------------- +# Code +#----------------------------------------------------------------------------- + class Quitter(object): """Simple class to handle exit, similar to Python 2.5's. @@ -40,4 +44,4 @@ class Quitter(object): # Repr MUST return a string, else display like pprint hooks get confused def __repr__(self): self.shell.ask_exit() - return 'Bye.' + return '' diff --git a/IPython/core/release.py b/IPython/core/release.py index 7195454..de540a9 100644 --- a/IPython/core/release.py +++ b/IPython/core/release.py @@ -21,9 +21,9 @@ name = 'ipython' # bdist_deb does not accept underscores (a Debian convention). development = True # change this to False to do a release -version_base = '0.11' +version_base = '0.11.alpha1' branch = 'ipython' -revision = '1346' +revision = '1223' if development: if branch == 'ipython': diff --git a/IPython/core/tests/test_iplib.py b/IPython/core/tests/test_iplib.py index f00204b..2fa6633 100644 --- a/IPython/core/tests/test_iplib.py +++ b/IPython/core/tests/test_iplib.py @@ -30,9 +30,9 @@ ip = get_ipython() @dec.parametric def test_reset(): """reset must clear most namespaces.""" - # The number of variables in the private user_config_ns is not zero, but it + # The number of variables in the private user_ns_hidden is not zero, but it # should be constant regardless of what we do - nvars_config_ns = len(ip.user_config_ns) + nvars_config_ns = len(ip.user_ns_hidden) # Check that reset runs without error ip.reset() @@ -51,7 +51,7 @@ def test_reset(): for ns in ip.ns_refs_table: if ns is ip.user_ns: nvars_expected = nvars_user_ns - elif ns is ip.user_config_ns: + elif ns is ip.user_ns_hidden: nvars_expected = nvars_config_ns else: nvars_expected = 0 diff --git a/IPython/core/tests/test_magic.py b/IPython/core/tests/test_magic.py index 758d6c3..efcecf7 100644 --- a/IPython/core/tests/test_magic.py +++ b/IPython/core/tests/test_magic.py @@ -8,19 +8,15 @@ from __future__ import absolute_import # Imports #----------------------------------------------------------------------------- -# stdlib import os import sys import tempfile import types from cStringIO import StringIO -# third-party import nose.tools as nt -# our own -from IPython.utils import genutils -from IPython.utils.platutils import find_cmd, get_long_path_name +from IPython.utils.path import get_long_path_name from IPython.testing import decorators as dec from IPython.testing import tools as tt diff --git a/IPython/core/tests/test_run.py b/IPython/core/tests/test_run.py index 79c167a..35d49ac 100644 --- a/IPython/core/tests/test_run.py +++ b/IPython/core/tests/test_run.py @@ -12,17 +12,12 @@ from __future__ import absolute_import # Imports #----------------------------------------------------------------------------- -# stdlib import os import sys import tempfile -# third-party import nose.tools as nt -# our own -from IPython.utils.platutils import find_cmd -from IPython.utils import genutils from IPython.testing import decorators as dec from IPython.testing import tools as tt @@ -142,10 +137,10 @@ class TestMagicRunSimple(tt.TempFileMixin): _ip.runlines('t = isinstance(f(), foo)') nt.assert_true(_ip.user_ns['t']) - # We have to skip these in win32 because genutils.getoutputerr() crashes, + # We have to skip these in win32 because getoutputerr() crashes, # due to the fact that subprocess does not support close_fds when # redirecting stdout/err. So unless someone who knows more tells us how to - # implement genutils.getoutputerr() in win32, we're stuck avoiding these. + # implement getoutputerr() in win32, we're stuck avoiding these. @dec.skip_win32 def test_obj_del(self): """Test that object's __del__ methods are called on exit.""" diff --git a/IPython/core/ultratb.py b/IPython/core/ultratb.py index c32fad9..d5b3e22 100644 --- a/IPython/core/ultratb.py +++ b/IPython/core/ultratb.py @@ -93,9 +93,10 @@ from inspect import getsourcefile, getfile, getmodule,\ from IPython.utils import PyColorize from IPython.core import debugger, ipapi from IPython.core.display_trap import DisplayTrap -from IPython.utils.ipstruct import Struct from IPython.core.excolors import exception_colors -from IPython.utils.genutils import Term, uniq_stable, error, info +from IPython.utils.data import uniq_stable +from IPython.utils.io import Term +from IPython.utils.warn import info, error # Globals # amount of space to put line numbers before verbose tracebacks @@ -384,7 +385,8 @@ class ListTB(TBTools): def __call__(self, etype, value, elist): Term.cout.flush() - Term.cerr.writeln(self.text(etype,value,elist)) + Term.cerr.write(self.text(etype,value,elist)) + Term.cerr.write('\n') def text(self, etype, value, elist, context=5): """Return a color formatted string with the traceback info. @@ -909,7 +911,8 @@ class VerboseTB(TBTools): (etype, evalue, etb) = info or sys.exc_info() self.tb = etb Term.cout.flush() - Term.cerr.writeln(self.text(etype, evalue, etb)) + Term.cerr.write(self.text(etype, evalue, etb)) + Term.cerr.write('\n') # Changed so an instance can just be called as VerboseTB_inst() and print # out the right info on its own. diff --git a/IPython/deathrow/GnuplotRuntime.py b/IPython/deathrow/GnuplotRuntime.py index e03cfe3..8d524e3 100644 --- a/IPython/deathrow/GnuplotRuntime.py +++ b/IPython/deathrow/GnuplotRuntime.py @@ -53,7 +53,7 @@ __all__ = ['Gnuplot','gp','gp_new','Data','File','Func','GridData', 'pm3d_config','eps_fix_bbox'] import os,tempfile,sys -from IPython.utils.genutils import getoutput +from IPython.utils.process import getoutput #--------------------------------------------------------------------------- # Notes on mouse support for Gnuplot.py diff --git a/IPython/deathrow/ipipe.py b/IPython/deathrow/ipipe.py index 74215bc..c440e6e 100644 --- a/IPython/deathrow/ipipe.py +++ b/IPython/deathrow/ipipe.py @@ -133,10 +133,10 @@ from IPython.external import simplegeneric from IPython.external import path try: - from IPython.utils import genutils + from IPython.utils.io import Term from IPython.utils import generics except ImportError: - genutils = None + Term = None generics = None from IPython.core import ipapi @@ -2168,7 +2168,7 @@ class idump(Display): self.datasepchar = "|" def display(self): - stream = genutils.Term.cout + stream = Term.cout allattrs = [] attrset = set() colwidths = {} diff --git a/IPython/deathrow/ipy_traits_completer.py b/IPython/deathrow/ipy_traits_completer.py index 2dfe620..ccf6d49 100644 --- a/IPython/deathrow/ipy_traits_completer.py +++ b/IPython/deathrow/ipy_traits_completer.py @@ -54,7 +54,7 @@ from enthought.traits import api as T # IPython imports from IPython.core.error import TryNext from IPython.core.ipapi import get as ipget -from IPython.utils.genutils import dir2 +from IPython.utils.dir2 import dir2 try: set except: diff --git a/IPython/deathrow/twshell.py b/IPython/deathrow/twshell.py index 6ad78fe..3270d65 100644 --- a/IPython/deathrow/twshell.py +++ b/IPython/deathrow/twshell.py @@ -14,7 +14,9 @@ from IPython.core.iplib import InteractiveShell from IPython.utils.ipstruct import Struct import Queue,thread,threading,signal from signal import signal, SIGINT -from IPython.utils.genutils import Term,warn,error,flag_calls, ask_yes_no +from IPython.utils.io import Term, ask_yes_no +from IPython.utils.warn import warn, error +from IPython.utils.decorators import flag_calls from IPython.core import shellglobals def install_gtk2(): diff --git a/IPython/extensions/pretty.py b/IPython/extensions/pretty.py index f6448e0..9484757 100644 --- a/IPython/extensions/pretty.py +++ b/IPython/extensions/pretty.py @@ -39,7 +39,7 @@ from IPython.core.error import TryNext from IPython.external import pretty from IPython.core.component import Component from IPython.utils.traitlets import Bool, List -from IPython.utils.genutils import Term +from IPython.utils.io import Term from IPython.utils.autoattr import auto_attr from IPython.utils.importstring import import_item @@ -128,9 +128,8 @@ class PrettyResultDisplay(Component): #----------------------------------------------------------------------------- -def load_ipython_extension(ip=None): +def load_ipython_extension(ip): """Load the extension in IPython as a hook.""" - if ip is None: ip = get_ipython() global _loaded if not _loaded: prd = PrettyResultDisplay(ip, name='pretty_result_display') diff --git a/IPython/external/mglob.py b/IPython/external/mglob.py index 1212c05..08f4194 100755 --- a/IPython/external/mglob.py +++ b/IPython/external/mglob.py @@ -213,7 +213,7 @@ def main(): print "\n".join(expand(sys.argv[1:])), def mglob_f(self, arg): - from IPython.utils.genutils import SList + from IPython.utils.text import SList if arg.strip(): return SList(expand(arg)) print "Please specify pattern!" diff --git a/IPython/frontend/linefrontendbase.py b/IPython/frontend/linefrontendbase.py index 18e0ba8..1dc570c 100644 --- a/IPython/frontend/linefrontendbase.py +++ b/IPython/frontend/linefrontendbase.py @@ -39,9 +39,10 @@ def common_prefix(strings): return prefix -#------------------------------------------------------------------------------- +#----------------------------------------------------------------------------- # Base class for the line-oriented front ends -#------------------------------------------------------------------------------- +#----------------------------------------------------------------------------- + class LineFrontEndBase(FrontEndBase): """ Concrete implementation of the FrontEndBase class. This is meant to be the base class behind all the frontend that are line-oriented, diff --git a/IPython/frontend/prefilterfrontend.py b/IPython/frontend/prefilterfrontend.py index 6cb2997..327ea0a 100644 --- a/IPython/frontend/prefilterfrontend.py +++ b/IPython/frontend/prefilterfrontend.py @@ -26,12 +26,12 @@ import os import re import __builtin__ -from IPython.core.ipapp import IPythonApp +from IPython.core.iplib import InteractiveShell from IPython.kernel.core.redirector_output_trap import RedirectorOutputTrap from IPython.kernel.core.sync_traceback_trap import SyncTracebackTrap -from IPython.utils.genutils import Term +from IPython.utils.io import Term from linefrontendbase import LineFrontEndBase, common_prefix @@ -50,9 +50,10 @@ def mk_system_call(system_call_function, command): my_system_call.__doc__ = "Calls %s" % command return my_system_call -#------------------------------------------------------------------------------- +#----------------------------------------------------------------------------- # Frontend class using ipython0 to do the prefiltering. -#------------------------------------------------------------------------------- +#----------------------------------------------------------------------------- + class PrefilterFrontEnd(LineFrontEndBase): """ Class that uses ipython0 to do prefilter the input, do the completion and the magics. @@ -65,19 +66,13 @@ class PrefilterFrontEnd(LineFrontEndBase): debug = False - def __init__(self, ipython0=None, argv=None, *args, **kwargs): + def __init__(self, ipython0=None, *args, **kwargs): """ Parameters ---------- ipython0: an optional ipython0 instance to use for command prefiltering and completion. - - argv : list, optional - Used as the instance's argv value. If not given, [] is used. """ - if argv is None: - argv = ['--no-banner'] - LineFrontEndBase.__init__(self, *args, **kwargs) self.shell.output_trap = RedirectorOutputTrap( out_callback=self.write, @@ -90,22 +85,19 @@ class PrefilterFrontEnd(LineFrontEndBase): # Start the ipython0 instance: self.save_output_hooks() if ipython0 is None: - # Instanciate an IPython0 interpreter to be able to use the + # Instanciate an IPython0 InteractiveShell to be able to use the # prefiltering. # Suppress all key input, to avoid waiting def my_rawinput(x=None): return '\n' old_rawinput = __builtin__.raw_input __builtin__.raw_input = my_rawinput - ipython0 = IPythonApp(argv=argv, - user_ns=self.shell.user_ns, - user_global_ns=self.shell.user_global_ns) - ipython0.initialize() + ipython0 = InteractiveShell( + parent=None, user_ns=self.shell.user_ns, + user_global_ns=self.shell.user_global_ns + ) __builtin__.raw_input = old_rawinput - # XXX This will need to be updated as we refactor things, but for now, - # the .shell attribute of the ipythonapp instance conforms to the old - # api. - self.ipython0 = ipython0.shell + self.ipython0 = ipython0 # Set the pager: self.ipython0.set_hook('show_in_pager', lambda s, string: self.write("\n" + string)) diff --git a/IPython/frontend/tests/test_asyncfrontendbase.py b/IPython/frontend/tests/test_asyncfrontendbase.py index b52659b..c009f98 100644 --- a/IPython/frontend/tests/test_asyncfrontendbase.py +++ b/IPython/frontend/tests/test_asyncfrontendbase.py @@ -1,14 +1,8 @@ # encoding: utf-8 - """This file contains unittests for the asyncfrontendbase module.""" -__docformat__ = "restructuredtext en" - -# Tell nose to skip this module -__test__ = {} - #--------------------------------------------------------------------------- -# Copyright (C) 2008 The IPython Development Team +# Copyright (C) 2008-2009 The IPython Development Team # # Distributed under the terms of the BSD License. The full license is in # the file COPYING, distributed as part of this software. diff --git a/IPython/frontend/tests/test_prefilterfrontend.py b/IPython/frontend/tests/test_prefilterfrontend.py index 81a51ca..432225a 100644 --- a/IPython/frontend/tests/test_prefilterfrontend.py +++ b/IPython/frontend/tests/test_prefilterfrontend.py @@ -21,7 +21,6 @@ from nose.tools import assert_equal from IPython.frontend.prefilterfrontend import PrefilterFrontEnd from IPython.testing.globalipapp import get_ipython -from IPython.testing.tools import default_argv #----------------------------------------------------------------------------- # Support utilities @@ -35,7 +34,7 @@ class TestPrefilterFrontEnd(PrefilterFrontEnd): def __init__(self): self.out = StringIO() - PrefilterFrontEnd.__init__(self,argv=default_argv()) + PrefilterFrontEnd.__init__(self) # Some more code for isolation (yeah, crazy) self._on_enter() self.out.flush() diff --git a/IPython/frontend/tests/test_process.py b/IPython/frontend/tests/test_process.py index 0b7adf8..ca49c26 100644 --- a/IPython/frontend/tests/test_process.py +++ b/IPython/frontend/tests/test_process.py @@ -47,6 +47,7 @@ def test_io(): assert result == test_string +@testdec.skip_win32 def test_kill(): """ Check that we can kill a process, and its subprocess. """ diff --git a/IPython/frontend/wx/console_widget.py b/IPython/frontend/wx/console_widget.py index b3b7245..970a61d 100644 --- a/IPython/frontend/wx/console_widget.py +++ b/IPython/frontend/wx/console_widget.py @@ -135,9 +135,10 @@ else: } -#------------------------------------------------------------------------------- +#----------------------------------------------------------------------------- # The console widget class -#------------------------------------------------------------------------------- +#----------------------------------------------------------------------------- + class ConsoleWidget(editwindow.EditWindow): """ Specialized styled text control view for console-like workflow. diff --git a/IPython/frontend/wx/ipythonx.py b/IPython/frontend/wx/ipythonx.py index d95d709..e4667b9 100644 --- a/IPython/frontend/wx/ipythonx.py +++ b/IPython/frontend/wx/ipythonx.py @@ -47,7 +47,7 @@ class IPythonXController(WxController): self._input_state = 'subprocess' self.write('\n', refresh=False) self.capture_output() - self.ipython0.shell.exit() + self.ipython0.exit() self.release_output() if not self.ipython0.exit_now: wx.CallAfter(self.new_prompt, diff --git a/IPython/gui/wx/ipshell_nonblocking.py b/IPython/gui/wx/ipshell_nonblocking.py index ee962b5..b8d4907 100755 --- a/IPython/gui/wx/ipshell_nonblocking.py +++ b/IPython/gui/wx/ipshell_nonblocking.py @@ -23,9 +23,8 @@ import os import locale from thread_ex import ThreadEx -import IPython -from IPython.core import iplib, ipapp -from IPython.utils import genutils +from IPython.core import iplib +from IPython.utils.io import Term ############################################################################## class _Helper(object): @@ -88,12 +87,10 @@ class NonBlockingIPShell(object): via raise_exc() ''' - def __init__(self, argv=[], user_ns={}, user_global_ns=None, + def __init__(self, user_ns={}, user_global_ns=None, cin=None, cout=None, cerr=None, ask_exit_handler=None): ''' - @param argv: Command line options for IPython - @type argv: list @param user_ns: User namespace. @type user_ns: dictionary @param user_global_ns: User global namespace. @@ -111,9 +108,9 @@ class NonBlockingIPShell(object): ''' #ipython0 initialisation self._IP = None - self.init_ipython0(argv, user_ns, user_global_ns, - cin, cout, cerr, - ask_exit_handler) + self.init_ipython0(user_ns, user_global_ns, + cin, cout, cerr, + ask_exit_handler) #vars used by _execute self._iter_more = 0 @@ -131,7 +128,7 @@ class NonBlockingIPShell(object): self._help_text = None self._add_button = None - def init_ipython0(self, argv=[], user_ns={}, user_global_ns=None, + def init_ipython0(self, user_ns={}, user_global_ns=None, cin=None, cout=None, cerr=None, ask_exit_handler=None): ''' Initialize an ipython0 instance ''' @@ -141,27 +138,22 @@ class NonBlockingIPShell(object): #only one instance can be instanciated else tehre will be #cin/cout/cerr clash... if cin: - genutils.Term.cin = cin + Term.cin = cin if cout: - genutils.Term.cout = cout + Term.cout = cout if cerr: - genutils.Term.cerr = cerr + Term.cerr = cerr excepthook = sys.excepthook #Hack to save sys.displayhook, because ipython seems to overwrite it... self.sys_displayhook_ori = sys.displayhook - - ipython0 = ipapp.IPythonApp(argv,user_ns=user_ns, - user_global_ns=user_global_ns) - ipython0.initialize() - self._IP = ipython0.shell - - ## self._IP = IPython.shell.make_IPython( - ## argv,user_ns=user_ns, - ## user_global_ns=user_global_ns, - ## embedded=True, - ## shell_class=IPython.shell.InteractiveShell) + ipython0 = iplib.InteractiveShell( + parent=None, config=None, + user_ns=user_ns, + user_global_ns=user_global_ns + ) + self._IP = ipython0 #we save ipython0 displayhook and we restore sys.displayhook self.displayhook = sys.displayhook @@ -185,12 +177,10 @@ class NonBlockingIPShell(object): #we replace the help command self._IP.user_ns['help'] = _Helper(self._pager_help) - #we disable cpase magic... until we found a way to use it properly. - from IPython.core import ipapi - ip = ipapi.get() + #we disable cpaste magic... until we found a way to use it properly. def bypass_magic(self, arg): print '%this magic is currently disabled.' - ip.define_magic('cpaste', bypass_magic) + ipython0.define_magic('cpaste', bypass_magic) import __builtin__ __builtin__.raw_input = self._raw_input @@ -471,7 +461,7 @@ class NonBlockingIPShell(object): ''' orig_stdout = sys.stdout - sys.stdout = genutils.Term.cout + sys.stdout = Term.cout #self.sys_displayhook_ori = sys.displayhook #sys.displayhook = self.displayhook diff --git a/IPython/gui/wx/ipython_history.py b/IPython/gui/wx/ipython_history.py index 91f8769..2e88ef0 100644 --- a/IPython/gui/wx/ipython_history.py +++ b/IPython/gui/wx/ipython_history.py @@ -11,6 +11,7 @@ __author__ = "Laurent Dufrechou" __email__ = "laurent.dufrechou _at_ gmail.com" __license__ = "BSD" #----------------------------------------- + class IPythonHistoryPanel(wx.Panel): def __init__(self, parent,flt_empty=True, diff --git a/IPython/kernel/client.py b/IPython/kernel/client.py index 872ec3a..13de59b 100644 --- a/IPython/kernel/client.py +++ b/IPython/kernel/client.py @@ -24,16 +24,25 @@ The main classes in this module are: #----------------------------------------------------------------------------- #----------------------------------------------------------------------------- -# Imports +# Warnings control #----------------------------------------------------------------------------- -from cStringIO import StringIO -import sys import warnings -# from IPython.utils import growl -# growl.start("IPython1 Client") +# Twisted generates annoying warnings with Python 2.6, as will do other code +# that imports 'sets' as of today +warnings.filterwarnings('ignore', 'the sets module is deprecated', + DeprecationWarning ) + +# This one also comes from Twisted +warnings.filterwarnings('ignore', 'the sha module is deprecated', + DeprecationWarning) + +#----------------------------------------------------------------------------- +# Imports +#----------------------------------------------------------------------------- +import sys from twisted.internet import reactor from twisted.internet.error import PotentialZombieWarning @@ -74,8 +83,6 @@ rit.setDaemon(True) rit.start() - - __all__ = [ 'MapTask', 'StringTask', diff --git a/IPython/kernel/clientconnector.py b/IPython/kernel/clientconnector.py index ffa55d4..dc40d05 100644 --- a/IPython/kernel/clientconnector.py +++ b/IPython/kernel/clientconnector.py @@ -1,6 +1,4 @@ -#!/usr/bin/env python # encoding: utf-8 - """Facilities for handling client connections to the controller.""" #----------------------------------------------------------------------------- @@ -20,6 +18,8 @@ import os from IPython.kernel.fcutil import ( Tub, find_furl, + is_valid_furl, + is_valid_furl_file, is_valid_furl_or_file, validate_furl_or_file, FURLError @@ -33,7 +33,7 @@ from IPython.kernel.twistedutil import ( sleep_deferred ) from IPython.utils.importstring import import_item -from IPython.utils.genutils import get_ipython_dir +from IPython.utils.path import get_ipython_dir from twisted.internet import defer from twisted.internet.defer import inlineCallbacks, returnValue @@ -66,18 +66,30 @@ class AsyncClientConnector(object): def _find_furl(self, profile='default', cluster_dir=None, furl_or_file=None, furl_file_name=None, ipython_dir=None): - """Find a FURL file by profile+ipython_dir or cluster dir. + """Find a FURL file. + + If successful, this returns a FURL file that exists on the file + system. The contents of the file have not been checked though. This + is because we often have to deal with FURL file whose buffers have + not been flushed. This raises an :exc:`~IPython.kernel.fcutil.FURLError` exception if a FURL file can't be found. + + This tries the following: + + 1. By the name ``furl_or_file``. + 2. By ``cluster_dir`` and ``furl_file_name``. + 3. By cluster profile with a default of ``default``. This uses + ``ipython_dir``. """ # Try by furl_or_file if furl_or_file is not None: - validate_furl_or_file(furl_or_file) - return furl_or_file + if is_valid_furl_or_file(furl_or_file): + return furl_or_file if furl_file_name is None: - raise FURLError('A furl_file_name must be provided') + raise FURLError('A furl_file_name must be provided if furl_or_file is not') # Try by cluster_dir if cluster_dir is not None: @@ -151,7 +163,7 @@ class AsyncClientConnector(object): The full path to a cluster directory. This is useful if profiles are not being used. furl_or_file : str - A furl or a filename containing a FURLK. This is useful if you + A furl or a filename containing a FURL. This is useful if you simply know the location of the FURL file. ipython_dir : str The location of the ipython_dir if different from the default. @@ -193,7 +205,7 @@ class AsyncClientConnector(object): The full path to a cluster directory. This is useful if profiles are not being used. furl_or_file : str - A furl or a filename containing a FURLK. This is useful if you + A furl or a filename containing a FURL. This is useful if you simply know the location of the FURL file. ipython_dir : str The location of the ipython_dir if different from the default. @@ -259,6 +271,9 @@ class AsyncClientConnector(object): profile, cluster_dir, furl_or_file, furl_file_name, ipython_dir ) + # If this succeeds, we know the furl file exists and has a .furl + # extension, but it could still be empty. That is checked each + # connection attempt. except FURLError: return defer.fail(failure.Failure()) @@ -349,7 +364,7 @@ class ClientConnector(object): The full path to a cluster directory. This is useful if profiles are not being used. furl_or_file : str - A furl or a filename containing a FURLK. This is useful if you + A furl or a filename containing a FURL. This is useful if you simply know the location of the FURL file. ipython_dir : str The location of the ipython_dir if different from the default. @@ -390,7 +405,7 @@ class ClientConnector(object): The full path to a cluster directory. This is useful if profiles are not being used. furl_or_file : str - A furl or a filename containing a FURLK. This is useful if you + A furl or a filename containing a FURL. This is useful if you simply know the location of the FURL file. ipython_dir : str The location of the ipython_dir if different from the default. diff --git a/IPython/kernel/clientinterfaces.py b/IPython/kernel/clientinterfaces.py index 248e511..362ae99 100644 --- a/IPython/kernel/clientinterfaces.py +++ b/IPython/kernel/clientinterfaces.py @@ -15,7 +15,7 @@ __docformat__ = "restructuredtext en" # Imports #------------------------------------------------------------------------------- -from zope.interface import Interface, implements +from zope.interface import Interface class IFCClientInterfaceProvider(Interface): diff --git a/IPython/kernel/clusterdir.py b/IPython/kernel/clusterdir.py index dcda0c6..63174eb 100755 --- a/IPython/kernel/clusterdir.py +++ b/IPython/kernel/clusterdir.py @@ -24,13 +24,16 @@ import warnings from twisted.python import log -from IPython.core import release from IPython.config.loader import PyFileConfigLoader -from IPython.core.application import Application +from IPython.core.application import Application, BaseAppConfigLoader from IPython.core.component import Component -from IPython.utils.genutils import get_ipython_dir, get_ipython_package_dir -from IPython.utils.traitlets import Unicode, Bool -from IPython.utils import genutils +from IPython.core.crashhandler import CrashHandler +from IPython.core import release +from IPython.utils.path import ( + get_ipython_package_dir, + expand_path +) +from IPython.utils.traitlets import Unicode #----------------------------------------------------------------------------- # Warnings control @@ -45,7 +48,7 @@ warnings.filterwarnings('ignore', 'the sha module is deprecated', DeprecationWarning) #----------------------------------------------------------------------------- -# Classes and functions +# Module errors #----------------------------------------------------------------------------- class ClusterDirError(Exception): @@ -56,6 +59,10 @@ class PIDFileError(Exception): pass +#----------------------------------------------------------------------------- +# Class for managing cluster directories +#----------------------------------------------------------------------------- + class ClusterDir(Component): """An object to manage the cluster directory and its resources. @@ -225,43 +232,109 @@ class ClusterDir(Component): The path of the cluster directory. This is expanded using :func:`IPython.utils.genutils.expand_path`. """ - cluster_dir = genutils.expand_path(cluster_dir) + cluster_dir = expand_path(cluster_dir) if not os.path.isdir(cluster_dir): raise ClusterDirError('Cluster directory not found: %s' % cluster_dir) return ClusterDir(cluster_dir) -# Default command line options for IPython cluster applications. -cl_args = ( - (('--ipython-dir',), dict( - dest='Global.ipython_dir',type=unicode, - help='Set to override default location of Global.ipython_dir.', - metavar='Global.ipython_dir') ), - (('-p', '--profile',), dict( - dest='Global.profile',type=unicode, - help= - """The string name of the profile to be used. This determines the name - of the cluster dir as: cluster_. The default profile is named - 'default'. The cluster directory is resolve this way if the - --cluster-dir option is not used.""", - metavar='Global.profile') ), - (('--cluster-dir',), dict( - dest='Global.cluster_dir',type=unicode, - help="""Set the cluster dir. This overrides the logic used by the - --profile option.""", - metavar='Global.cluster_dir') ), - (('--work-dir',), dict( - dest='Global.work_dir',type=unicode, - help='Set the working dir for the process.', - metavar='Global.work_dir') ), - (('--clean-logs',), dict( - dest='Global.clean_logs', action='store_true', - help='Delete old log flies before starting.') ), - (('--no-clean-logs',), dict( - dest='Global.clean_logs', action='store_false', - help="Don't Delete old log flies before starting.") ), - ) +#----------------------------------------------------------------------------- +# Command line options +#----------------------------------------------------------------------------- + +class ClusterDirConfigLoader(BaseAppConfigLoader): + + def _add_cluster_profile(self, parser): + paa = parser.add_argument + paa('-p', '--profile', + dest='Global.profile',type=unicode, + help= + """The string name of the profile to be used. This determines the name + of the cluster dir as: cluster_. The default profile is named + 'default'. The cluster directory is resolve this way if the + --cluster-dir option is not used.""", + metavar='Global.profile') + + def _add_cluster_dir(self, parser): + paa = parser.add_argument + paa('--cluster-dir', + dest='Global.cluster_dir',type=unicode, + help="""Set the cluster dir. This overrides the logic used by the + --profile option.""", + metavar='Global.cluster_dir') + + def _add_work_dir(self, parser): + paa = parser.add_argument + paa('--work-dir', + dest='Global.work_dir',type=unicode, + help='Set the working dir for the process.', + metavar='Global.work_dir') + + def _add_clean_logs(self, parser): + paa = parser.add_argument + paa('--clean-logs', + dest='Global.clean_logs', action='store_true', + help='Delete old log flies before starting.') + + def _add_no_clean_logs(self, parser): + paa = parser.add_argument + paa('--no-clean-logs', + dest='Global.clean_logs', action='store_false', + help="Don't Delete old log flies before starting.") + + def _add_arguments(self): + super(ClusterDirConfigLoader, self)._add_arguments() + self._add_cluster_profile(self.parser) + self._add_cluster_dir(self.parser) + self._add_work_dir(self.parser) + self._add_clean_logs(self.parser) + self._add_no_clean_logs(self.parser) + + +#----------------------------------------------------------------------------- +# Crash handler for this application +#----------------------------------------------------------------------------- + + +_message_template = """\ +Oops, $self.app_name crashed. We do our best to make it stable, but... + +A crash report was automatically generated with the following information: + - A verbatim copy of the crash traceback. + - Data on your current $self.app_name configuration. +It was left in the file named: +\t'$self.crash_report_fname' +If you can email this file to the developers, the information in it will help +them in understanding and correcting the problem. + +You can mail it to: $self.contact_name at $self.contact_email +with the subject '$self.app_name Crash Report'. + +If you want to do it now, the following command will work (under Unix): +mail -s '$self.app_name Crash Report' $self.contact_email < $self.crash_report_fname + +To ensure accurate tracking of this issue, please file a report about it at: +$self.bug_tracker +""" + +class ClusterDirCrashHandler(CrashHandler): + """sys.excepthook for IPython itself, leaves a detailed report on disk.""" + + message_template = _message_template + + def __init__(self, app): + contact_name = release.authors['Brian'][0] + contact_email = release.authors['Brian'][1] + bug_tracker = 'https://bugs.launchpad.net/ipython/+filebug' + super(ClusterDirCrashHandler,self).__init__( + app, contact_name, contact_email, bug_tracker + ) + + +#----------------------------------------------------------------------------- +# Main application +#----------------------------------------------------------------------------- class ApplicationWithClusterDir(Application): """An application that puts everything into a cluster directory. @@ -282,10 +355,10 @@ class ApplicationWithClusterDir(Application): dir and named the value of the ``config_file_name`` class attribute. """ + command_line_loader = ClusterDirConfigLoader + crash_handler_class = ClusterDirCrashHandler auto_create_cluster_dir = True - cl_arguments = Application.cl_arguments + cl_args - def create_default_config(self): super(ApplicationWithClusterDir, self).create_default_config() self.default_config.Global.profile = u'default' @@ -316,7 +389,7 @@ class ApplicationWithClusterDir(Application): cluster_dir = self.command_line_config.Global.cluster_dir except AttributeError: cluster_dir = self.default_config.Global.cluster_dir - cluster_dir = genutils.expand_path(cluster_dir) + cluster_dir = expand_path(cluster_dir) try: self.cluster_dir_obj = ClusterDir.find_cluster_dir(cluster_dir) except ClusterDirError: @@ -368,15 +441,17 @@ class ApplicationWithClusterDir(Application): def find_config_file_name(self): """Find the config file name for this application.""" # For this type of Application it should be set as a class attribute. - if not hasattr(self, 'config_file_name'): + if not hasattr(self, 'default_config_file_name'): self.log.critical("No config filename found") + else: + self.config_file_name = self.default_config_file_name def find_config_file_paths(self): - # Include our own config directory last, so that users can still find - # our shipped copies of builtin config files even if they don't have - # them in their ipython cluster directory. + # Set the search path to to the cluster directory. We should NOT + # include IPython.config.default here as the default config files + # are ALWAYS automatically moved to the cluster directory. conf_dir = os.path.join(get_ipython_package_dir(), 'config', 'default') - self.config_file_paths = (self.cluster_dir, conf_dir) + self.config_file_paths = (self.cluster_dir,) def pre_construct(self): # The log and security dirs were set earlier, but here we put them @@ -389,7 +464,7 @@ class ApplicationWithClusterDir(Application): pdir = self.cluster_dir_obj.pid_dir self.pid_dir = config.Global.pid_dir = pdir self.log.info("Cluster directory set to: %s" % self.cluster_dir) - config.Global.work_dir = unicode(genutils.expand_path(config.Global.work_dir)) + config.Global.work_dir = unicode(expand_path(config.Global.work_dir)) # Change to the working directory. We do this just before construct # is called so all the components there have the right working dir. self.to_work_dir() @@ -461,3 +536,4 @@ class ApplicationWithClusterDir(Application): return pid else: raise PIDFileError('pid file not found: %s' % pid_file) + diff --git a/IPython/kernel/contexts.py b/IPython/kernel/contexts.py deleted file mode 100644 index 2e9daa3..0000000 --- a/IPython/kernel/contexts.py +++ /dev/null @@ -1,141 +0,0 @@ -# encoding: utf-8 -# -*- test-case-name: IPython.kernel.test.test_contexts -*- -"""Context managers for IPython. - -Python 2.5 introduced the `with` statement, which is based on the context -manager protocol. This module offers a few context managers for common cases, -which can also be useful as templates for writing new, application-specific -managers. -""" - -__docformat__ = "restructuredtext en" - -#------------------------------------------------------------------------------- -# Copyright (C) 2008 The IPython Development Team -# -# Distributed under the terms of the BSD License. The full license is in -# the file COPYING, distributed as part of this software. -#------------------------------------------------------------------------------- - -#------------------------------------------------------------------------------- -# Imports -#------------------------------------------------------------------------------- - -import linecache -import sys - -from twisted.internet.error import ConnectionRefusedError - -from IPython.core.ultratb import _fixed_getinnerframes, findsource -from IPython.core import ipapi - -from IPython.kernel import error - -#--------------------------------------------------------------------------- -# Utility functions needed by all context managers. -#--------------------------------------------------------------------------- - -def remote(): - """Raises a special exception meant to be caught by context managers. - """ - m = 'Special exception to stop local execution of parallel code.' - raise error.StopLocalExecution(m) - - -def strip_whitespace(source,require_remote=True): - """strip leading whitespace from input source. - - :Parameters: - - """ - remote_mark = 'remote()' - # Expand tabs to avoid any confusion. - wsource = [l.expandtabs(4) for l in source] - # Detect the indentation level - done = False - for line in wsource: - if line.isspace(): - continue - for col,char in enumerate(line): - if char != ' ': - done = True - break - if done: - break - # Now we know how much leading space there is in the code. Next, we - # extract up to the first line that has less indentation. - # WARNINGS: we skip comments that may be misindented, but we do NOT yet - # detect triple quoted strings that may have flush left text. - for lno,line in enumerate(wsource): - lead = line[:col] - if lead.isspace(): - continue - else: - if not lead.lstrip().startswith('#'): - break - # The real 'with' source is up to lno - src_lines = [l[col:] for l in wsource[:lno+1]] - - # Finally, check that the source's first non-comment line begins with the - # special call 'remote()' - if require_remote: - for nline,line in enumerate(src_lines): - if line.isspace() or line.startswith('#'): - continue - if line.startswith(remote_mark): - break - else: - raise ValueError('%s call missing at the start of code' % - remote_mark) - out_lines = src_lines[nline+1:] - else: - # If the user specified that the remote() call wasn't mandatory - out_lines = src_lines - - # src = ''.join(out_lines) # dbg - #print 'SRC:\n<<<<<<<>>>>>>>\n%s<<<<<>>>>>>' % src # dbg - return ''.join(out_lines) - -class RemoteContextBase(object): - def __init__(self): - self.ip = ipapi.get() - - def _findsource_file(self,f): - linecache.checkcache() - s = findsource(f.f_code) - lnum = f.f_lineno - wsource = s[0][f.f_lineno:] - return strip_whitespace(wsource) - - def _findsource_ipython(self,f): - from IPython.core import ipapi - self.ip = ipapi.get() - buf = self.ip.input_hist_raw[-1].splitlines()[1:] - wsource = [l+'\n' for l in buf ] - - return strip_whitespace(wsource) - - def findsource(self,frame): - local_ns = frame.f_locals - global_ns = frame.f_globals - if frame.f_code.co_filename == '': - src = self._findsource_ipython(frame) - else: - src = self._findsource_file(frame) - return src - - def __enter__(self): - raise NotImplementedError - - def __exit__ (self, etype, value, tb): - if issubclass(etype,error.StopLocalExecution): - return True - -class RemoteMultiEngine(RemoteContextBase): - def __init__(self,mec): - self.mec = mec - RemoteContextBase.__init__(self) - - def __enter__(self): - src = self.findsource(sys._getframe(1)) - return self.mec.execute(src) diff --git a/IPython/kernel/controllerservice.py b/IPython/kernel/controllerservice.py index 10ad03b..e8e30f4 100644 --- a/IPython/kernel/controllerservice.py +++ b/IPython/kernel/controllerservice.py @@ -37,20 +37,18 @@ __docformat__ = "restructuredtext en" # Imports #------------------------------------------------------------------------------- -import os, sys +import os from twisted.application import service -from twisted.internet import defer, reactor -from twisted.python import log, components +from twisted.python import log from zope.interface import Interface, implements, Attribute -import zope.interface as zi from IPython.kernel.engineservice import \ IEngineCore, \ IEngineSerialized, \ IEngineQueued -from IPython.utils.genutils import get_ipython_dir +from IPython.utils.path import get_ipython_dir from IPython.kernel import codeutil #------------------------------------------------------------------------------- diff --git a/IPython/kernel/core/prompts.py b/IPython/kernel/core/prompts.py index 1759df7..fc583ea 100644 --- a/IPython/kernel/core/prompts.py +++ b/IPython/kernel/core/prompts.py @@ -21,6 +21,8 @@ __docformat__ = "restructuredtext en" # Required modules import __builtin__ +import os +import re import socket import sys @@ -30,7 +32,8 @@ from IPython.external.Itpl import ItplNS from IPython.utils import coloransi from IPython.core import release from IPython.core.error import TryNext -from IPython.utils.genutils import * +from IPython.utils.io import Term +from IPython.utils.warn import warn import IPython.utils.generics #**************************************************************************** @@ -240,7 +243,7 @@ class BasePrompt(object): This must be called every time the color settings change, because the prompt_specials global may have changed.""" - import os,time # needed in locals for prompt string handling + import os, time # needed in locals for prompt string handling loc = locals() self.p_str = ItplNS('%s%s%s' % ('${self.sep}${self.col_p}', diff --git a/IPython/kernel/core/tests/test_redirectors.py b/IPython/kernel/core/tests/test_redirectors.py index fe48371..cc54449 100644 --- a/IPython/kernel/core/tests/test_redirectors.py +++ b/IPython/kernel/core/tests/test_redirectors.py @@ -22,15 +22,15 @@ import sys from twisted.trial import unittest +from IPython.testing import decorators_trial as dec + #----------------------------------------------------------------------------- # Tests #----------------------------------------------------------------------------- class TestRedirector(unittest.TestCase): - if sys.platform == 'win32': - skip = True - + @dec.skip_win32 def test_redirector(self): """Checks that the redirector can be used to do synchronous capture. """ @@ -51,6 +51,7 @@ class TestRedirector(unittest.TestCase): result2 = "".join("%ic\n%i\n" %(i, i) for i in range(10)) self.assertEquals(result1, result2) + @dec.skip_win32 def test_redirector_output_trap(self): """Check the greedy trapping behavior of the traps. diff --git a/IPython/kernel/engineconnector.py b/IPython/kernel/engineconnector.py index dfa983f..36a12c4 100644 --- a/IPython/kernel/engineconnector.py +++ b/IPython/kernel/engineconnector.py @@ -17,8 +17,7 @@ import os import cPickle as pickle -from twisted.python import log, failure -from twisted.internet import defer +from twisted.python import log from twisted.internet.defer import inlineCallbacks, returnValue from IPython.kernel.fcutil import find_furl, validate_furl_or_file diff --git a/IPython/kernel/enginefc.py b/IPython/kernel/enginefc.py index ebeff5c..fb6333b 100644 --- a/IPython/kernel/enginefc.py +++ b/IPython/kernel/enginefc.py @@ -19,40 +19,36 @@ __docformat__ = "restructuredtext en" # Imports #------------------------------------------------------------------------------- -import os, time import cPickle as pickle from twisted.python import components, log, failure -from twisted.python.failure import Failure -from twisted.internet import defer, reactor, threads -from twisted.internet.interfaces import IProtocolFactory -from zope.interface import Interface, implements, Attribute +from twisted.internet import defer, threads +from zope.interface import Interface, implements from twisted.internet.base import DelayedCall DelayedCall.debug = True -from foolscap import Referenceable, DeadReferenceError +try: + from foolscap.api import Referenceable, DeadReferenceError +except ImportError: + from foolscap import Referenceable, DeadReferenceError from foolscap.referenceable import RemoteReference from IPython.kernel.pbutil import packageFailure, unpackageFailure -from IPython.kernel.util import printer -from IPython.kernel.twistedutil import gatherBoth -from IPython.kernel import newserialized -from IPython.kernel.error import ProtocolError -from IPython.kernel import controllerservice from IPython.kernel.controllerservice import IControllerBase -from IPython.kernel.engineservice import \ - IEngineBase, \ - IEngineQueued, \ - EngineService, \ +from IPython.kernel.engineservice import ( + IEngineBase, + IEngineQueued, StrictDict -from IPython.kernel.pickleutil import \ - can, \ - canDict, \ - canSequence, \ - uncan, \ - uncanDict, \ +) +from IPython.kernel.pickleutil import ( + can, + canDict, + canSequence, + uncan, + uncanDict, uncanSequence +) #------------------------------------------------------------------------------- diff --git a/IPython/kernel/error.py b/IPython/kernel/error.py index b5f6d24..dd1e1c5 100644 --- a/IPython/kernel/error.py +++ b/IPython/kernel/error.py @@ -17,6 +17,9 @@ __test__ = {} #------------------------------------------------------------------------------- # Imports #------------------------------------------------------------------------------- + +from twisted.python import failure + from IPython.kernel.core import error #------------------------------------------------------------------------------- @@ -26,6 +29,7 @@ from IPython.kernel.core import error class KernelError(error.IPythonError): pass + class NotDefined(KernelError): def __init__(self, name): self.name = name @@ -36,78 +40,102 @@ class NotDefined(KernelError): __str__ = __repr__ + class QueueCleared(KernelError): pass + class IdInUse(KernelError): pass + class ProtocolError(KernelError): pass + class ConnectionError(KernelError): pass + class InvalidEngineID(KernelError): pass - + + class NoEnginesRegistered(KernelError): pass - + + class InvalidClientID(KernelError): pass - + + class InvalidDeferredID(KernelError): pass - + + class SerializationError(KernelError): pass - + + class MessageSizeError(KernelError): pass - + + class PBMessageSizeError(MessageSizeError): pass - + + class ResultNotCompleted(KernelError): pass - + + class ResultAlreadyRetrieved(KernelError): pass - + class ClientError(KernelError): pass + class TaskAborted(KernelError): pass + class TaskTimeout(KernelError): pass + class NotAPendingResult(KernelError): pass + class UnpickleableException(KernelError): pass + class AbortedPendingDeferredError(KernelError): pass + class InvalidProperty(KernelError): pass + class MissingBlockArgument(KernelError): pass + class StopLocalExecution(KernelError): pass + class SecurityError(KernelError): pass + class FileTimeoutError(KernelError): pass + class TaskRejectError(KernelError): """Exception to raise when a task should be rejected by an engine. @@ -122,6 +150,7 @@ class TaskRejectError(KernelError): properties don't have to be managed or tested by the controller. """ + class CompositeError(KernelError): def __init__(self, message, elist): Exception.__init__(self, *(message, elist)) @@ -176,9 +205,8 @@ class CompositeError(KernelError): else: raise et, ev, etb -def collect_exceptions(rlist, method): - from twisted.python import failure +def collect_exceptions(rlist, method): elist = [] for r in rlist: if isinstance(r, failure.Failure): @@ -203,5 +231,4 @@ def collect_exceptions(rlist, method): raise CompositeError(msg, elist) except CompositeError, e: raise e - diff --git a/IPython/kernel/fcutil.py b/IPython/kernel/fcutil.py index 4de4e89..ec4de2e 100644 --- a/IPython/kernel/fcutil.py +++ b/IPython/kernel/fcutil.py @@ -23,16 +23,19 @@ import tempfile from twisted.internet import reactor, defer from twisted.python import log -from foolscap import Tub, UnauthenticatedTub +import foolscap +try: + from foolscap.api import Tub, UnauthenticatedTub +except ImportError: + from foolscap import Tub, UnauthenticatedTub from IPython.config.loader import Config - from IPython.kernel.configobjfactory import AdaptedConfiguredObjectFactory - from IPython.kernel.error import SecurityError -from IPython.utils.traitlets import Int, Str, Bool, Instance from IPython.utils.importstring import import_item +from IPython.utils.path import expand_path +from IPython.utils.traitlets import Int, Str, Bool, Instance #----------------------------------------------------------------------------- # Code @@ -57,17 +60,17 @@ class FURLError(Exception): def check_furl_file_security(furl_file, secure): """Remove the old furl_file if changing security modes.""" + furl_file = expand_path(furl_file) if os.path.isfile(furl_file): - f = open(furl_file, 'r') - oldfurl = f.read().strip() - f.close() + with open(furl_file, 'r') as f: + oldfurl = f.read().strip() if (oldfurl.startswith('pb://') and not secure) or (oldfurl.startswith('pbu://') and secure): os.remove(furl_file) def is_secure(furl): """Is the given FURL secure or not.""" - if is_valid(furl): + if is_valid_furl(furl): if furl.startswith("pb://"): return True elif furl.startswith("pbu://"): @@ -76,26 +79,45 @@ def is_secure(furl): raise FURLError("invalid FURL: %s" % furl) -def is_valid(furl): +def is_valid_furl(furl): """Is the str a valid FURL or not.""" if isinstance(furl, str): if furl.startswith("pb://") or furl.startswith("pbu://"): return True + else: + return False else: return False +def is_valid_furl_file(furl_or_file): + """See if furl_or_file exists and contains a valid FURL. + + This doesn't try to read the contents because often we have to validate + FURL files that are created, but don't yet have a FURL written to them. + """ + if isinstance(furl_or_file, (str, unicode)): + path, furl_filename = os.path.split(furl_or_file) + if os.path.isdir(path) and furl_filename.endswith('.furl'): + return True + return False + + def find_furl(furl_or_file): - """Find, validate and return a FURL in a string or file.""" - if isinstance(furl_or_file, str): - if is_valid(furl_or_file): - return furl_or_file - if os.path.isfile(furl_or_file): + """Find, validate and return a FURL in a string or file. + + This calls :func:`IPython.utils.path.expand_path` on the argument to + properly handle ``~`` and ``$`` variables in the path. + """ + if is_valid_furl(furl_or_file): + return furl_or_file + furl_or_file = expand_path(furl_or_file) + if is_valid_furl_file(furl_or_file): with open(furl_or_file, 'r') as f: furl = f.read().strip() - if is_valid(furl): + if is_valid_furl(furl): return furl - raise FURLError("Not a valid FURL or FURL file: %s" % furl_or_file) + raise FURLError("Not a valid FURL or FURL file: %r" % furl_or_file) def is_valid_furl_or_file(furl_or_file): @@ -106,17 +128,14 @@ def is_valid_furl_or_file(furl_or_file): if the FURL file exists or to read its contents. This is useful for cases where auto re-connection is being used. """ - if isinstance(furl_or_file, str): - if is_valid(furl_or_file): - return True - if isinstance(furl_or_file, (str, unicode)): - path, furl_filename = os.path.split(furl_or_file) - if os.path.isdir(path) and furl_filename.endswith('.furl'): - return True - return False + if is_valid_furl(furl_or_file) or is_valid_furl_file(furl_or_file): + return True + else: + return False def validate_furl_or_file(furl_or_file): + """Like :func:`is_valid_furl_or_file`, but raises an error.""" if not is_valid_furl_or_file(furl_or_file): raise FURLError('Not a valid FURL or FURL file: %r' % furl_or_file) @@ -265,9 +284,13 @@ class FCServiceFactory(AdaptedConfiguredObjectFactory): """Register the reference with the FURL file. The FURL file is created and then moved to make sure that when the - file appears, the buffer has been flushed and the file closed. + file appears, the buffer has been flushed and the file closed. This + is not done if we are re-using FURLS however. """ - temp_furl_file = get_temp_furlfile(furl_file) - self.tub.registerReference(ref, furlFile=temp_furl_file) - os.rename(temp_furl_file, furl_file) + if self.reuse_furls: + self.tub.registerReference(ref, furlFile=furl_file) + else: + temp_furl_file = get_temp_furlfile(furl_file) + self.tub.registerReference(ref, furlFile=temp_furl_file) + os.rename(temp_furl_file, furl_file) diff --git a/IPython/kernel/ipclusterapp.py b/IPython/kernel/ipclusterapp.py index f30a882..57b7675 100755 --- a/IPython/kernel/ipclusterapp.py +++ b/IPython/kernel/ipclusterapp.py @@ -22,161 +22,203 @@ import signal if os.name=='posix': from twisted.scripts._twistd_unix import daemonize -from IPython.core import release -from IPython.external.argparse import ArgumentParser -from IPython.config.loader import ArgParseConfigLoader, NoConfigDefault -from IPython.utils.importstring import import_item +from twisted.internet import reactor, defer +from twisted.python import log, failure + +from IPython.external.argparse import ArgumentParser, SUPPRESS +from IPython.utils.importstring import import_item from IPython.kernel.clusterdir import ( - ApplicationWithClusterDir, ClusterDirError, PIDFileError + ApplicationWithClusterDir, ClusterDirConfigLoader, + ClusterDirError, PIDFileError ) -from twisted.internet import reactor, defer -from twisted.python import log, failure - #----------------------------------------------------------------------------- -# The ipcluster application +# Module level variables #----------------------------------------------------------------------------- +default_config_file_name = u'ipcluster_config.py' + + +_description = """\ +Start an IPython cluster for parallel computing.\n\n + +An IPython cluster consists of 1 controller and 1 or more engines. +This command automates the startup of these processes using a wide +range of startup methods (SSH, local processes, PBS, mpiexec, +Windows HPC Server 2008). To start a cluster with 4 engines on your +local host simply do 'ipcluster start -n 4'. For more complex usage +you will typically do 'ipcluster create -p mycluster', then edit +configuration files, followed by 'ipcluster start -p mycluster -n 4'. +""" + + # Exit codes for ipcluster # This will be the exit code if the ipcluster appears to be running because # a .pid file exists ALREADY_STARTED = 10 + # This will be the exit code if ipcluster stop is run, but there is not .pid # file to be found. ALREADY_STOPPED = 11 -class IPClusterCLLoader(ArgParseConfigLoader): +#----------------------------------------------------------------------------- +# Command line options +#----------------------------------------------------------------------------- + - def _add_other_arguments(self): +class IPClusterAppConfigLoader(ClusterDirConfigLoader): + + def _add_arguments(self): + # Don't call ClusterDirConfigLoader._add_arguments as we don't want + # its defaults on self.parser. Instead, we will put those on + # default options on our subparsers. + # This has all the common options that all subcommands use - parent_parser1 = ArgumentParser(add_help=False, - argument_default=NoConfigDefault) - parent_parser1.add_argument('--ipython-dir', - dest='Global.ipython_dir',type=unicode, - help='Set to override default location of Global.ipython_dir.', - metavar='Global.ipython_dir') - parent_parser1.add_argument('--log-level', - dest="Global.log_level",type=int, - help='Set the log level (0,10,20,30,40,50). Default is 30.', - metavar='Global.log_level') + parent_parser1 = ArgumentParser( + add_help=False, + argument_default=SUPPRESS + ) + self._add_ipython_dir(parent_parser1) + self._add_log_level(parent_parser1) # This has all the common options that other subcommands use - parent_parser2 = ArgumentParser(add_help=False, - argument_default=NoConfigDefault) - parent_parser2.add_argument('-p','--profile', - dest='Global.profile',type=unicode, - help='The string name of the profile to be used. This determines ' - 'the name of the cluster dir as: cluster_. The default profile ' - 'is named "default". The cluster directory is resolve this way ' - 'if the --cluster-dir option is not used.', - metavar='Global.profile') - parent_parser2.add_argument('--cluster-dir', - dest='Global.cluster_dir',type=unicode, - help='Set the cluster dir. This overrides the logic used by the ' - '--profile option.', - metavar='Global.cluster_dir'), - parent_parser2.add_argument('--work-dir', - dest='Global.work_dir',type=unicode, - help='Set the working dir for the process.', - metavar='Global.work_dir') - parent_parser2.add_argument('--log-to-file', - action='store_true', dest='Global.log_to_file', - help='Log to a file in the log directory (default is stdout)' + parent_parser2 = ArgumentParser( + add_help=False, + argument_default=SUPPRESS ) + self._add_cluster_profile(parent_parser2) + self._add_cluster_dir(parent_parser2) + self._add_work_dir(parent_parser2) + paa = parent_parser2.add_argument + paa('--log-to-file', + action='store_true', dest='Global.log_to_file', + help='Log to a file in the log directory (default is stdout)') + # Create the object used to create the subparsers. subparsers = self.parser.add_subparsers( dest='Global.subcommand', title='ipcluster subcommands', - description='ipcluster has a variety of subcommands. ' - 'The general way of running ipcluster is "ipcluster ' - ' [options]""', - help='For more help, type "ipcluster -h"') + description= + """ipcluster has a variety of subcommands. The general way of + running ipcluster is 'ipcluster [options]'. To get help + on a particular subcommand do 'ipcluster -h'.""" + # help="For more help, type 'ipcluster -h'", + ) + # The "list" subcommand parser parser_list = subparsers.add_parser( 'list', - help='List all clusters in cwd and ipython_dir.', - parents=[parent_parser1] + parents=[parent_parser1], + argument_default=SUPPRESS, + help="List all clusters in cwd and ipython_dir.", + description= + """List all available clusters, by cluster directory, that can + be found in the current working directly or in the ipython + directory. Cluster directories are named using the convention + 'cluster_'.""" ) + # The "create" subcommand parser parser_create = subparsers.add_parser( 'create', - help='Create a new cluster directory.', - parents=[parent_parser1, parent_parser2] + parents=[parent_parser1, parent_parser2], + argument_default=SUPPRESS, + help="Create a new cluster directory.", + description= + """Create an ipython cluster directory by its profile name or + cluster directory path. Cluster directories contain + configuration, log and security related files and are named + using the convention 'cluster_'. By default they are + located in your ipython directory. Once created, you will + probably need to edit the configuration files in the cluster + directory to configure your cluster. Most users will create a + cluster directory by profile name, + 'ipcluster create -p mycluster', which will put the directory + in '/cluster_mycluster'. + """ ) - parser_create.add_argument( - '--reset-config', + paa = parser_create.add_argument + paa('--reset-config', dest='Global.reset_config', action='store_true', - default=NoConfigDefault, - help='Recopy the default config files to the cluster directory. ' - 'You will loose any modifications you have made to these files.' - ) + help= + """Recopy the default config files to the cluster directory. + You will loose any modifications you have made to these files.""") + # The "start" subcommand parser parser_start = subparsers.add_parser( 'start', - help='Start a cluster.', - parents=[parent_parser1, parent_parser2] + parents=[parent_parser1, parent_parser2], + argument_default=SUPPRESS, + help="Start a cluster.", + description= + """Start an ipython cluster by its profile name or cluster + directory. Cluster directories contain configuration, log and + security related files and are named using the convention + 'cluster_' and should be creating using the 'start' + subcommand of 'ipcluster'. If your cluster directory is in + the cwd or the ipython directory, you can simply refer to it + using its profile name, 'ipcluster start -n 4 -p `, + otherwise use the '--cluster-dir' option. + """ ) - parser_start.add_argument( - '-n', '--number', + paa = parser_start.add_argument + paa('-n', '--number', type=int, dest='Global.n', help='The number of engines to start.', - metavar='Global.n' - ) - parser_start.add_argument('--clean-logs', + metavar='Global.n') + paa('--clean-logs', dest='Global.clean_logs', action='store_true', - help='Delete old log flies before starting.', - ) - parser_start.add_argument('--no-clean-logs', + help='Delete old log flies before starting.') + paa('--no-clean-logs', dest='Global.clean_logs', action='store_false', - help="Don't delete old log flies before starting.", - ) - parser_start.add_argument('--daemon', + help="Don't delete old log flies before starting.") + paa('--daemon', dest='Global.daemonize', action='store_true', - help='Daemonize the ipcluster program. This implies --log-to-file', - ) - parser_start.add_argument('--no-daemon', + help='Daemonize the ipcluster program. This implies --log-to-file') + paa('--no-daemon', dest='Global.daemonize', action='store_false', - help="Dont't daemonize the ipcluster program.", - ) + help="Dont't daemonize the ipcluster program.") - parser_start = subparsers.add_parser( + # The "stop" subcommand parser + parser_stop = subparsers.add_parser( 'stop', - help='Stop a cluster.', - parents=[parent_parser1, parent_parser2] + parents=[parent_parser1, parent_parser2], + argument_default=SUPPRESS, + help="Stop a running cluster.", + description= + """Stop a running ipython cluster by its profile name or cluster + directory. Cluster directories are named using the convention + 'cluster_'. If your cluster directory is in + the cwd or the ipython directory, you can simply refer to it + using its profile name, 'ipcluster stop -p `, otherwise + use the '--cluster-dir' option. + """ ) - parser_start.add_argument('--signal', + paa = parser_stop.add_argument + paa('--signal', dest='Global.signal', type=int, help="The signal number to use in stopping the cluster (default=2).", - metavar="Global.signal", - ) - + metavar="Global.signal") -default_config_file_name = u'ipcluster_config.py' - -_description = """Start an IPython cluster for parallel computing.\n\n - -An IPython cluster consists of 1 controller and 1 or more engines. -This command automates the startup of these processes using a wide -range of startup methods (SSH, local processes, PBS, mpiexec, -Windows HPC Server 2008). To start a cluster with 4 engines on your -local host simply do "ipcluster start -n 4". For more complex usage -you will typically do "ipcluster create -p mycluster", then edit -configuration files, followed by "ipcluster start -p mycluster -n 4". -""" +#----------------------------------------------------------------------------- +# Main application +#----------------------------------------------------------------------------- class IPClusterApp(ApplicationWithClusterDir): name = u'ipcluster' description = _description - config_file_name = default_config_file_name + usage = None + command_line_loader = IPClusterAppConfigLoader + default_config_file_name = default_config_file_name default_log_level = logging.INFO auto_create_cluster_dir = False @@ -192,13 +234,6 @@ class IPClusterApp(ApplicationWithClusterDir): self.default_config.Global.signal = 2 self.default_config.Global.daemonize = False - def create_command_line_config(self): - """Create and return a command line config loader.""" - return IPClusterCLLoader( - description=self.description, - version=release.version - ) - def find_resources(self): subcommand = self.command_line_config.Global.subcommand if subcommand=='list': @@ -361,8 +396,11 @@ class IPClusterApp(ApplicationWithClusterDir): log.msg('Unexpected error in ipcluster:') log.msg(r.getTraceback()) log.msg("IPython cluster: stopping") - self.stop_engines() - self.stop_controller() + # These return deferreds. We are not doing anything with them + # but we are holding refs to them as a reminder that they + # do return deferreds. + d1 = self.stop_engines() + d2 = self.stop_controller() # Wait a few seconds to let things shut down. reactor.callLater(4.0, reactor.stop) @@ -449,6 +487,7 @@ class IPClusterApp(ApplicationWithClusterDir): # old .pid files. self.remove_pid_file() + def launch_new_instance(): """Create and run the IPython cluster.""" app = IPClusterApp() diff --git a/IPython/kernel/ipcontrollerapp.py b/IPython/kernel/ipcontrollerapp.py index 7a2e9d2..37cf583 100755 --- a/IPython/kernel/ipcontrollerapp.py +++ b/IPython/kernel/ipcontrollerapp.py @@ -18,20 +18,40 @@ The IPython controller application. from __future__ import with_statement import copy -import os import sys from twisted.application import service from twisted.internet import reactor from twisted.python import log -from IPython.config.loader import Config, NoConfigDefault -from IPython.core import release -from IPython.core.application import Application +from IPython.config.loader import Config from IPython.kernel import controllerservice -from IPython.kernel.clusterdir import ApplicationWithClusterDir -from IPython.kernel.fcutil import FCServiceFactory -from IPython.utils.traitlets import Str, Instance, Unicode +from IPython.kernel.clusterdir import ( + ApplicationWithClusterDir, + ClusterDirConfigLoader +) +from IPython.kernel.fcutil import FCServiceFactory, FURLError +from IPython.utils.traitlets import Instance, Unicode + + +#----------------------------------------------------------------------------- +# Module level variables +#----------------------------------------------------------------------------- + + +#: The default config file name for this application +default_config_file_name = u'ipcontroller_config.py' + + +_description = """Start the IPython controller for parallel computing. + +The IPython controller provides a gateway between the IPython engines and +clients. The controller needs to be started before the engines and can be +configured using command line options or using a cluster directory. Cluster +directories contain config, log and security files and are usually located in +your .ipython directory and named as "cluster_". See the --profile +and --cluster-dir options for details. +""" #----------------------------------------------------------------------------- # Default interfaces @@ -92,108 +112,96 @@ class FCEngineServiceFactory(FCServiceFactory): #----------------------------------------------------------------------------- -# The main application +# Command line options #----------------------------------------------------------------------------- -cl_args = ( - # Client config - (('--client-ip',), dict( - type=str, dest='FCClientServiceFactory.ip', - help='The IP address or hostname the controller will listen on for ' - 'client connections.', - metavar='FCClientServiceFactory.ip') - ), - (('--client-port',), dict( - type=int, dest='FCClientServiceFactory.port', - help='The port the controller will listen on for client connections. ' - 'The default is to use 0, which will autoselect an open port.', - metavar='FCClientServiceFactory.port') - ), - (('--client-location',), dict( - type=str, dest='FCClientServiceFactory.location', - help='The hostname or IP that clients should connect to. This does ' - 'not control which interface the controller listens on. Instead, this ' - 'determines the hostname/IP that is listed in the FURL, which is how ' - 'clients know where to connect. Useful if the controller is listening ' - 'on multiple interfaces.', - metavar='FCClientServiceFactory.location') - ), - # Engine config - (('--engine-ip',), dict( - type=str, dest='FCEngineServiceFactory.ip', - help='The IP address or hostname the controller will listen on for ' - 'engine connections.', - metavar='FCEngineServiceFactory.ip') - ), - (('--engine-port',), dict( - type=int, dest='FCEngineServiceFactory.port', - help='The port the controller will listen on for engine connections. ' - 'The default is to use 0, which will autoselect an open port.', - metavar='FCEngineServiceFactory.port') - ), - (('--engine-location',), dict( - type=str, dest='FCEngineServiceFactory.location', - help='The hostname or IP that engines should connect to. This does ' - 'not control which interface the controller listens on. Instead, this ' - 'determines the hostname/IP that is listed in the FURL, which is how ' - 'engines know where to connect. Useful if the controller is listening ' - 'on multiple interfaces.', - metavar='FCEngineServiceFactory.location') - ), - # Global config - (('--log-to-file',), dict( - action='store_true', dest='Global.log_to_file', - help='Log to a file in the log directory (default is stdout)') - ), - (('-r','--reuse-furls'), dict( - action='store_true', dest='Global.reuse_furls', - help='Try to reuse all FURL files. If this is not set all FURL files ' - 'are deleted before the controller starts. This must be set if ' - 'specific ports are specified by --engine-port or --client-port.') - ), - (('--no-secure',), dict( - action='store_false', dest='Global.secure', - help='Turn off SSL encryption for all connections.') - ), - (('--secure',), dict( - action='store_true', dest='Global.secure', - help='Turn off SSL encryption for all connections.') - ) -) +class IPControllerAppConfigLoader(ClusterDirConfigLoader): + + def _add_arguments(self): + super(IPControllerAppConfigLoader, self)._add_arguments() + paa = self.parser.add_argument + # Client config + paa('--client-ip', + type=str, dest='FCClientServiceFactory.ip', + help='The IP address or hostname the controller will listen on for ' + 'client connections.', + metavar='FCClientServiceFactory.ip') + paa('--client-port', + type=int, dest='FCClientServiceFactory.port', + help='The port the controller will listen on for client connections. ' + 'The default is to use 0, which will autoselect an open port.', + metavar='FCClientServiceFactory.port') + paa('--client-location',), dict( + type=str, dest='FCClientServiceFactory.location', + help='The hostname or IP that clients should connect to. This does ' + 'not control which interface the controller listens on. Instead, this ' + 'determines the hostname/IP that is listed in the FURL, which is how ' + 'clients know where to connect. Useful if the controller is listening ' + 'on multiple interfaces.', + metavar='FCClientServiceFactory.location') + # Engine config + paa('--engine-ip', + type=str, dest='FCEngineServiceFactory.ip', + help='The IP address or hostname the controller will listen on for ' + 'engine connections.', + metavar='FCEngineServiceFactory.ip') + paa('--engine-port', + type=int, dest='FCEngineServiceFactory.port', + help='The port the controller will listen on for engine connections. ' + 'The default is to use 0, which will autoselect an open port.', + metavar='FCEngineServiceFactory.port') + paa('--engine-location', + type=str, dest='FCEngineServiceFactory.location', + help='The hostname or IP that engines should connect to. This does ' + 'not control which interface the controller listens on. Instead, this ' + 'determines the hostname/IP that is listed in the FURL, which is how ' + 'engines know where to connect. Useful if the controller is listening ' + 'on multiple interfaces.', + metavar='FCEngineServiceFactory.location') + # Global config + paa('--log-to-file', + action='store_true', dest='Global.log_to_file', + help='Log to a file in the log directory (default is stdout)') + paa('-r','--reuse-furls', + action='store_true', dest='Global.reuse_furls', + help='Try to reuse all FURL files. If this is not set all FURL files ' + 'are deleted before the controller starts. This must be set if ' + 'specific ports are specified by --engine-port or --client-port.') + paa('--no-secure', + action='store_false', dest='Global.secure', + help='Turn off SSL encryption for all connections.') + paa('--secure', + action='store_true', dest='Global.secure', + help='Turn off SSL encryption for all connections.') -_description = """Start the IPython controller for parallel computing. - -The IPython controller provides a gateway between the IPython engines and -clients. The controller needs to be started before the engines and can be -configured using command line options or using a cluster directory. Cluster -directories contain config, log and security files and are usually located in -your .ipython directory and named as "cluster_". See the --profile -and --cluster-dir options for details. -""" - -default_config_file_name = u'ipcontroller_config.py' +#----------------------------------------------------------------------------- +# The main application +#----------------------------------------------------------------------------- class IPControllerApp(ApplicationWithClusterDir): name = u'ipcontroller' description = _description - config_file_name = default_config_file_name + command_line_loader = IPControllerAppConfigLoader + default_config_file_name = default_config_file_name auto_create_cluster_dir = True - cl_arguments = Application.cl_arguments + cl_args def create_default_config(self): super(IPControllerApp, self).create_default_config() - self.default_config.Global.reuse_furls = False - self.default_config.Global.secure = True + # Don't set defaults for Global.secure or Global.reuse_furls + # as those are set in a component. self.default_config.Global.import_statements = [] self.default_config.Global.clean_logs = True - def post_load_command_line_config(self): - # Now setup reuse_furls - c = self.command_line_config + def pre_construct(self): + super(IPControllerApp, self).pre_construct() + c = self.master_config + # The defaults for these are set in FCClientServiceFactory and + # FCEngineServiceFactory, so we only set them here if the global + # options have be set to override the class level defaults. if hasattr(c.Global, 'reuse_furls'): c.FCClientServiceFactory.reuse_furls = c.Global.reuse_furls c.FCEngineServiceFactory.reuse_furls = c.Global.reuse_furls @@ -216,11 +224,19 @@ class IPControllerApp(ApplicationWithClusterDir): controller_service = controllerservice.ControllerService() controller_service.setServiceParent(self.main_service) # The client tub and all its refereceables - csfactory = FCClientServiceFactory(self.master_config, controller_service) + try: + csfactory = FCClientServiceFactory(self.master_config, controller_service) + except FURLError, e: + log.err(e) + self.exit(0) client_service = csfactory.create() client_service.setServiceParent(self.main_service) # The engine tub - esfactory = FCEngineServiceFactory(self.master_config, controller_service) + try: + esfactory = FCEngineServiceFactory(self.master_config, controller_service) + except FURLError, e: + log.err(e) + self.exit(0) engine_service = esfactory.create() engine_service.setServiceParent(self.main_service) diff --git a/IPython/kernel/ipengineapp.py b/IPython/kernel/ipengineapp.py index dbd3e35..61f010a 100755 --- a/IPython/kernel/ipengineapp.py +++ b/IPython/kernel/ipengineapp.py @@ -22,39 +22,21 @@ from twisted.application import service from twisted.internet import reactor from twisted.python import log -from IPython.core.application import Application -from IPython.kernel.clusterdir import ApplicationWithClusterDir +from IPython.kernel.clusterdir import ( + ApplicationWithClusterDir, + ClusterDirConfigLoader +) from IPython.kernel.engineconnector import EngineConnector from IPython.kernel.engineservice import EngineService from IPython.kernel.fcutil import Tub from IPython.utils.importstring import import_item #----------------------------------------------------------------------------- -# The main application +# Module level variables #----------------------------------------------------------------------------- -cl_args = ( - # Controller config - (('--furl-file',), dict( - type=unicode, dest='Global.furl_file', - help='The full location of the file containing the FURL of the ' - 'controller. If this is not given, the FURL file must be in the ' - 'security directory of the cluster directory. This location is ' - 'resolved using the --profile and --app-dir options.', - metavar='Global.furl_file') - ), - # MPI - (('--mpi',), dict( - type=str, dest='MPI.use', - help='How to enable MPI (mpi4py, pytrilinos, or empty string to disable).', - metavar='MPI.use') - ), - # Global config - (('--log-to-file',), dict( - action='store_true', dest='Global.log_to_file', - help='Log to a file in the log directory (default is stdout)') - ) -) +#: The default config file name for this application +default_config_file_name = u'ipengine_config.py' mpi4py_init = """from mpi4py import MPI as mpi @@ -62,6 +44,7 @@ mpi.size = mpi.COMM_WORLD.Get_size() mpi.rank = mpi.COMM_WORLD.Get_rank() """ + pytrilinos_init = """from PyTrilinos import Epetra class SimpleStruct: pass @@ -71,9 +54,6 @@ mpi.size = 0 """ -default_config_file_name = u'ipengine_config.py' - - _description = """Start an IPython engine for parallel computing.\n\n IPython engines run in parallel and perform computations on behalf of a client @@ -84,14 +64,47 @@ usually located in your .ipython directory and named as "cluster_". See the --profile and --cluster-dir options for details. """ +#----------------------------------------------------------------------------- +# Command line options +#----------------------------------------------------------------------------- + + +class IPEngineAppConfigLoader(ClusterDirConfigLoader): + + def _add_arguments(self): + super(IPEngineAppConfigLoader, self)._add_arguments() + paa = self.parser.add_argument + # Controller config + paa('--furl-file', + type=unicode, dest='Global.furl_file', + help='The full location of the file containing the FURL of the ' + 'controller. If this is not given, the FURL file must be in the ' + 'security directory of the cluster directory. This location is ' + 'resolved using the --profile and --app-dir options.', + metavar='Global.furl_file') + # MPI + paa('--mpi', + type=str, dest='MPI.use', + help='How to enable MPI (mpi4py, pytrilinos, or empty string to disable).', + metavar='MPI.use') + # Global config + paa('--log-to-file', + action='store_true', dest='Global.log_to_file', + help='Log to a file in the log directory (default is stdout)') + + +#----------------------------------------------------------------------------- +# Main application +#----------------------------------------------------------------------------- + class IPEngineApp(ApplicationWithClusterDir): name = u'ipengine' description = _description - config_file_name = default_config_file_name + command_line_loader = IPEngineAppConfigLoader + default_config_file_name = default_config_file_name auto_create_cluster_dir = True - cl_arguments = Application.cl_arguments + cl_args def create_default_config(self): super(IPEngineApp, self).create_default_config() diff --git a/IPython/kernel/launcher.py b/IPython/kernel/launcher.py index 2f41211..43cf057 100644 --- a/IPython/kernel/launcher.py +++ b/IPython/kernel/launcher.py @@ -21,11 +21,15 @@ import sys from IPython.core.component import Component from IPython.external import Itpl -from IPython.utils.traitlets import Str, Int, List, Unicode, Enum -from IPython.utils.platutils import find_cmd -from IPython.kernel.twistedutil import gatherBoth, make_deferred, sleep_deferred +from IPython.utils.traitlets import Str, Int, List, Unicode +from IPython.utils.path import get_ipython_module_path +from IPython.utils.process import find_cmd, pycmd2argv, FindCmdError +from IPython.kernel.twistedutil import ( + gatherBoth, + make_deferred, + sleep_deferred +) from IPython.kernel.winhpcjob import ( - WinHPCJob, WinHPCTask, IPControllerTask, IPEngineTask, IPControllerJob, IPEngineSetJob ) @@ -38,46 +42,23 @@ from twisted.internet.error import ProcessDone, ProcessTerminated from twisted.python import log from twisted.python.failure import Failure + #----------------------------------------------------------------------------- -# Utilities +# Paths to the kernel apps #----------------------------------------------------------------------------- -def find_controller_cmd(): - """Find the command line ipcontroller program in a cross platform way.""" - if sys.platform == 'win32': - # This logic is needed because the ipcontroller script doesn't - # always get installed in the same way or in the same location. - from IPython.kernel import ipcontrollerapp - script_location = ipcontrollerapp.__file__.replace('.pyc', '.py') - # The -u option here turns on unbuffered output, which is required - # on Win32 to prevent wierd conflict and problems with Twisted. - # Also, use sys.executable to make sure we are picking up the - # right python exe. - cmd = [sys.executable, '-u', script_location] - else: - # ipcontroller has to be on the PATH in this case. - cmd = ['ipcontroller'] - return cmd - - -def find_engine_cmd(): - """Find the command line ipengine program in a cross platform way.""" - if sys.platform == 'win32': - # This logic is needed because the ipengine script doesn't - # always get installed in the same way or in the same location. - from IPython.kernel import ipengineapp - script_location = ipengineapp.__file__.replace('.pyc', '.py') - # The -u option here turns on unbuffered output, which is required - # on Win32 to prevent wierd conflict and problems with Twisted. - # Also, use sys.executable to make sure we are picking up the - # right python exe. - cmd = [sys.executable, '-u', script_location] - else: - # ipcontroller has to be on the PATH in this case. - cmd = ['ipengine'] - return cmd +ipcluster_cmd_argv = pycmd2argv(get_ipython_module_path( + 'IPython.kernel.ipclusterapp' +)) + +ipengine_cmd_argv = pycmd2argv(get_ipython_module_path( + 'IPython.kernel.ipengineapp' +)) +ipcontroller_cmd_argv = pycmd2argv(get_ipython_module_path( + 'IPython.kernel.ipcontrollerapp' +)) #----------------------------------------------------------------------------- # Base launchers and errors @@ -333,7 +314,7 @@ class LocalProcessLauncher(BaseLauncher): class LocalControllerLauncher(LocalProcessLauncher): """Launch a controller as a regular external process.""" - controller_cmd = List(find_controller_cmd(), config=True) + controller_cmd = List(ipcontroller_cmd_argv, config=True) # Command line arguments to ipcontroller. controller_args = List(['--log-to-file','--log-level', '40'], config=True) @@ -351,7 +332,7 @@ class LocalControllerLauncher(LocalProcessLauncher): class LocalEngineLauncher(LocalProcessLauncher): """Launch a single engine as a regular externall process.""" - engine_cmd = List(find_engine_cmd(), config=True) + engine_cmd = List(ipengine_cmd_argv, config=True) # Command line arguments for ipengine. engine_args = List( ['--log-to-file','--log-level', '40'], config=True @@ -462,7 +443,7 @@ class MPIExecLauncher(LocalProcessLauncher): class MPIExecControllerLauncher(MPIExecLauncher): """Launch a controller using mpiexec.""" - controller_cmd = List(find_controller_cmd(), config=True) + controller_cmd = List(ipcontroller_cmd_argv, config=True) # Command line arguments to ipcontroller. controller_args = List(['--log-to-file','--log-level', '40'], config=True) n = Int(1, config=False) @@ -481,7 +462,7 @@ class MPIExecControllerLauncher(MPIExecLauncher): class MPIExecEngineSetLauncher(MPIExecLauncher): - engine_cmd = List(find_engine_cmd(), config=True) + engine_cmd = List(ipengine_cmd_argv, config=True) # Command line arguments for ipengine. engine_args = List( ['--log-to-file','--log-level', '40'], config=True @@ -557,7 +538,10 @@ class SSHEngineSetLauncher(BaseLauncher): # This is only used on Windows. def find_job_cmd(): if os.name=='nt': - return find_cmd('job') + try: + return find_cmd('job') + except FindCmdError: + return 'job' else: return 'job' @@ -831,28 +815,10 @@ class PBSEngineSetLauncher(PBSLauncher): #----------------------------------------------------------------------------- -def find_ipcluster_cmd(): - """Find the command line ipcluster program in a cross platform way.""" - if sys.platform == 'win32': - # This logic is needed because the ipcluster script doesn't - # always get installed in the same way or in the same location. - from IPython.kernel import ipclusterapp - script_location = ipclusterapp.__file__.replace('.pyc', '.py') - # The -u option here turns on unbuffered output, which is required - # on Win32 to prevent wierd conflict and problems with Twisted. - # Also, use sys.executable to make sure we are picking up the - # right python exe. - cmd = [sys.executable, '-u', script_location] - else: - # ipcontroller has to be on the PATH in this case. - cmd = ['ipcluster'] - return cmd - - class IPClusterLauncher(LocalProcessLauncher): """Launch the ipcluster program in an external process.""" - ipcluster_cmd = List(find_ipcluster_cmd(), config=True) + ipcluster_cmd = List(ipcluster_cmd_argv, config=True) # Command line arguments to pass to ipcluster. ipcluster_args = List( ['--clean-logs', '--log-to-file', '--log-level', '40'], config=True) diff --git a/IPython/kernel/map.py b/IPython/kernel/map.py index f9ce2f8..6d2d9ea 100644 --- a/IPython/kernel/map.py +++ b/IPython/kernel/map.py @@ -21,7 +21,7 @@ __docformat__ = "restructuredtext en" import types -from IPython.utils.genutils import flatten as genutil_flatten +from IPython.utils.data import flatten as utils_flatten #------------------------------------------------------------------------------- # Figure out which array packages are present and their array types @@ -87,7 +87,7 @@ class Map: return m['module'].concatenate(listOfPartitions) # Next try for Python sequence types if isinstance(testObject, (types.ListType, types.TupleType)): - return genutil_flatten(listOfPartitions) + return utils_flatten(listOfPartitions) # If we have scalars, just return listOfPartitions return listOfPartitions diff --git a/IPython/kernel/mapper.py b/IPython/kernel/mapper.py index e732b53..28f2545 100644 --- a/IPython/kernel/mapper.py +++ b/IPython/kernel/mapper.py @@ -18,8 +18,7 @@ __docformat__ = "restructuredtext en" from types import FunctionType from zope.interface import Interface, implements from IPython.kernel.task import MapTask -from IPython.kernel.twistedutil import DeferredList, gatherBoth -from IPython.kernel.util import printer +from IPython.kernel.twistedutil import gatherBoth from IPython.kernel.error import collect_exceptions #---------------------------------------------------------------------------- diff --git a/IPython/kernel/multiengine.py b/IPython/kernel/multiengine.py index bdeba67..bb520b6 100644 --- a/IPython/kernel/multiengine.py +++ b/IPython/kernel/multiengine.py @@ -27,24 +27,17 @@ __docformat__ = "restructuredtext en" # Imports #------------------------------------------------------------------------------- -from new import instancemethod -from types import FunctionType - -from twisted.application import service from twisted.internet import defer, reactor from twisted.python import log, components, failure -from zope.interface import Interface, implements, Attribute +from zope.interface import Interface, implements -from IPython.utils import growl -from IPython.kernel.util import printer from IPython.kernel.twistedutil import gatherBoth -from IPython.kernel import map as Map from IPython.kernel import error from IPython.kernel.pendingdeferred import PendingDeferredManager, two_phase -from IPython.kernel.controllerservice import \ - ControllerAdapterBase, \ - ControllerService, \ +from IPython.kernel.controllerservice import ( + ControllerAdapterBase, IControllerBase +) #------------------------------------------------------------------------------- diff --git a/IPython/kernel/multiengineclient.py b/IPython/kernel/multiengineclient.py index ccdb51b..c206578 100644 --- a/IPython/kernel/multiengineclient.py +++ b/IPython/kernel/multiengineclient.py @@ -17,13 +17,17 @@ __docformat__ = "restructuredtext en" #------------------------------------------------------------------------------- import sys -import linecache import warnings from twisted.python import components from twisted.python.failure import Failure from zope.interface import Interface, implements, Attribute +try: + from foolscap.api import DeadReferenceError +except ImportError: + from foolscap import DeadReferenceError + from IPython.utils.coloransi import TermColors from IPython.kernel.twistedutil import blockingCallFromThread @@ -306,85 +310,6 @@ class InteractiveMultiEngineClient(object): def __len__(self): """Return the number of available engines.""" return len(self.get_ids()) - - #--------------------------------------------------------------------------- - # Make this a context manager for with - #--------------------------------------------------------------------------- - - def findsource_file(self,f): - linecache.checkcache() - s = findsource(f.f_code) # findsource is not defined! - lnum = f.f_lineno - wsource = s[0][f.f_lineno:] - return strip_whitespace(wsource) - - def findsource_ipython(self,f): - from IPython.core import ipapi - self.ip = ipapi.get() - wsource = [l+'\n' for l in - self.ip.input_hist_raw[-1].splitlines()[1:]] - return strip_whitespace(wsource) - - def __enter__(self): - f = sys._getframe(1) - local_ns = f.f_locals - global_ns = f.f_globals - if f.f_code.co_filename == '': - s = self.findsource_ipython(f) - else: - s = self.findsource_file(f) - - self._with_context_result = self.execute(s) - - def __exit__ (self, etype, value, tb): - if issubclass(etype,error.StopLocalExecution): - return True - - -def remote(): - m = 'Special exception to stop local execution of parallel code.' - raise error.StopLocalExecution(m) - -def strip_whitespace(source): - # Expand tabs to avoid any confusion. - wsource = [l.expandtabs(4) for l in source] - # Detect the indentation level - done = False - for line in wsource: - if line.isspace(): - continue - for col,char in enumerate(line): - if char != ' ': - done = True - break - if done: - break - # Now we know how much leading space there is in the code. Next, we - # extract up to the first line that has less indentation. - # WARNINGS: we skip comments that may be misindented, but we do NOT yet - # detect triple quoted strings that may have flush left text. - for lno,line in enumerate(wsource): - lead = line[:col] - if lead.isspace(): - continue - else: - if not lead.lstrip().startswith('#'): - break - # The real 'with' source is up to lno - src_lines = [l[col:] for l in wsource[:lno+1]] - - # Finally, check that the source's first non-comment line begins with the - # special call 'remote()' - for nline,line in enumerate(src_lines): - if line.isspace() or line.startswith('#'): - continue - if 'remote()' in line: - break - else: - raise ValueError('remote() call missing at the start of code') - src = ''.join(src_lines[nline+1:]) - #print 'SRC:\n<<<<<<<>>>>>>>\n%s<<<<<>>>>>>' % src # dbg - return src #------------------------------------------------------------------------------- @@ -444,18 +369,31 @@ class FullBlockingMultiEngineClient(InteractiveMultiEngineClient): def _findTargetsAndBlock(self, targets=None, block=None): return self._findTargets(targets), self._findBlock(block) - + + def _bcft(self, *args, **kwargs): + try: + result = blockingCallFromThread(*args, **kwargs) + except DeadReferenceError: + raise error.ConnectionError( + """A connection error has occurred in trying to connect to the + controller. This is usually caused by the controller dying or + being restarted. To resolve this issue try recreating the + multiengine client.""" + ) + else: + return result + def _blockFromThread(self, function, *args, **kwargs): block = kwargs.get('block', None) if block is None: raise error.MissingBlockArgument("'block' keyword argument is missing") - result = blockingCallFromThread(function, *args, **kwargs) + result = self._bcft(function, *args, **kwargs) if not block: result = PendingResult(self, result) return result def get_pending_deferred(self, deferredID, block): - return blockingCallFromThread(self.smultiengine.get_pending_deferred, deferredID, block) + return self._bcft(self.smultiengine.get_pending_deferred, deferredID, block) def barrier(self, pendingResults): """Synchronize a set of `PendingResults`. @@ -505,7 +443,7 @@ class FullBlockingMultiEngineClient(InteractiveMultiEngineClient): controller. This method allows the user to clear out all un-retrieved results on the controller. """ - r = blockingCallFromThread(self.smultiengine.clear_pending_deferreds) + r = self._bcft(self.smultiengine.clear_pending_deferreds) return r clear_pending_results = flush @@ -529,7 +467,7 @@ class FullBlockingMultiEngineClient(InteractiveMultiEngineClient): at a later time. """ targets, block = self._findTargetsAndBlock(targets, block) - result = blockingCallFromThread(self.smultiengine.execute, lines, + result = self._bcft(self.smultiengine.execute, lines, targets=targets, block=block) if block: result = ResultList(result) @@ -647,7 +585,7 @@ class FullBlockingMultiEngineClient(InteractiveMultiEngineClient): at a later time. """ targets, block = self._findTargetsAndBlock(targets, block) - result = blockingCallFromThread(self.smultiengine.get_result, i, targets=targets, block=block) + result = self._bcft(self.smultiengine.get_result, i, targets=targets, block=block) if block: result = ResultList(result) else: @@ -773,7 +711,7 @@ class FullBlockingMultiEngineClient(InteractiveMultiEngineClient): """ Returns the ids of currently registered engines. """ - result = blockingCallFromThread(self.smultiengine.get_ids) + result = self._bcft(self.smultiengine.get_ids) return result #--------------------------------------------------------------------------- diff --git a/IPython/kernel/multienginefc.py b/IPython/kernel/multienginefc.py index 30de28d..ff77ef4 100644 --- a/IPython/kernel/multienginefc.py +++ b/IPython/kernel/multienginefc.py @@ -22,12 +22,14 @@ from types import FunctionType from zope.interface import Interface, implements from twisted.internet import defer -from twisted.python import components, failure, log +from twisted.python import components, failure -from foolscap import Referenceable +try: + from foolscap.api import Referenceable +except ImportError: + from foolscap import Referenceable from IPython.kernel import error -from IPython.kernel.util import printer from IPython.kernel import map as Map from IPython.kernel.parallelfunction import ParallelFunction from IPython.kernel.mapper import ( @@ -36,14 +38,15 @@ from IPython.kernel.mapper import ( IMapper ) from IPython.kernel.twistedutil import gatherBoth -from IPython.kernel.multiengine import (MultiEngine, +from IPython.kernel.multiengine import ( IMultiEngine, IFullSynchronousMultiEngine, ISynchronousMultiEngine) -from IPython.kernel.multiengineclient import wrapResultList from IPython.kernel.pendingdeferred import PendingDeferredManager -from IPython.kernel.pickleutil import (can, canDict, - canSequence, uncan, uncanDict, uncanSequence) +from IPython.kernel.pickleutil import ( + canDict, + canSequence, uncanDict, uncanSequence +) from IPython.kernel.clientinterfaces import ( IFCClientInterfaceProvider, diff --git a/IPython/kernel/pbutil.py b/IPython/kernel/pbutil.py index 6ce8050..da98b82 100644 --- a/IPython/kernel/pbutil.py +++ b/IPython/kernel/pbutil.py @@ -19,7 +19,6 @@ import cPickle as pickle from twisted.python.failure import Failure from twisted.python import failure -import threading, sys from IPython.kernel import pbconfig from IPython.kernel.error import PBMessageSizeError, UnpickleableException @@ -58,7 +57,7 @@ def unpackageFailure(r): result = pickle.loads(r[8:]) except pickle.PickleError: return failure.Failure( \ - FailureUnpickleable("Could not unpickle failure.")) + UnpickleableException("Could not unpickle failure.")) else: return result return r diff --git a/IPython/kernel/pendingdeferred.py b/IPython/kernel/pendingdeferred.py index 91abdfb..b741f36 100644 --- a/IPython/kernel/pendingdeferred.py +++ b/IPython/kernel/pendingdeferred.py @@ -22,15 +22,11 @@ __docformat__ = "restructuredtext en" # Imports #------------------------------------------------------------------------------- -from twisted.application import service -from twisted.internet import defer, reactor -from twisted.python import log, components, failure -from zope.interface import Interface, implements, Attribute +from twisted.internet import defer +from twisted.python import failure -from IPython.kernel.twistedutil import gatherBoth from IPython.kernel import error from IPython.external import guid -from IPython.utils import growl class PendingDeferredManager(object): """A class to track pending deferreds. diff --git a/IPython/kernel/pickleutil.py b/IPython/kernel/pickleutil.py index 087a61c..faed1c5 100644 --- a/IPython/kernel/pickleutil.py +++ b/IPython/kernel/pickleutil.py @@ -16,7 +16,6 @@ __docformat__ = "restructuredtext en" #------------------------------------------------------------------------------- from types import FunctionType -from twisted.python import log class CannedObject(object): pass diff --git a/IPython/kernel/task.py b/IPython/kernel/task.py index 924d052..ec0c70d 100644 --- a/IPython/kernel/task.py +++ b/IPython/kernel/task.py @@ -19,19 +19,18 @@ __docformat__ = "restructuredtext en" # Tell nose to skip the testing of this module __test__ = {} -import copy, time +import time from types import FunctionType -import zope.interface as zi, string +import zope.interface as zi from twisted.internet import defer, reactor from twisted.python import components, log, failure -from IPython.kernel.util import printer from IPython.kernel import engineservice as es, error from IPython.kernel import controllerservice as cs -from IPython.kernel.twistedutil import gatherBoth, DeferredList +from IPython.kernel.twistedutil import DeferredList -from IPython.kernel.pickleutil import can, uncan, CannedFunction +from IPython.kernel.pickleutil import can, uncan #----------------------------------------------------------------------------- # Definition of the Task objects diff --git a/IPython/kernel/taskclient.py b/IPython/kernel/taskclient.py index 69225fb..a9dcad9 100644 --- a/IPython/kernel/taskclient.py +++ b/IPython/kernel/taskclient.py @@ -19,7 +19,12 @@ __docformat__ = "restructuredtext en" #------------------------------------------------------------------------------- from zope.interface import Interface, implements -from twisted.python import components, log +from twisted.python import components + +try: + from foolscap.api import DeadReferenceError +except ImportError: + from foolscap import DeadReferenceError from IPython.kernel.twistedutil import blockingCallFromThread from IPython.kernel import task, error @@ -58,7 +63,20 @@ class BlockingTaskClient(object): def __init__(self, task_controller): self.task_controller = task_controller self.block = True - + + def _bcft(self, *args, **kwargs): + try: + result = blockingCallFromThread(*args, **kwargs) + except DeadReferenceError: + raise error.ConnectionError( + """A connection error has occurred in trying to connect to the + controller. This is usually caused by the controller dying or + being restarted. To resolve this issue try recreating the + task client.""" + ) + else: + return result + def run(self, task, block=False): """Run a task on the `TaskController`. @@ -71,7 +89,7 @@ class BlockingTaskClient(object): :Returns: The int taskid of the submitted task. Pass this to `get_task_result` to get the `TaskResult` object. """ - tid = blockingCallFromThread(self.task_controller.run, task) + tid = self._bcft(self.task_controller.run, task) if block: return self.get_task_result(tid, block=True) else: @@ -89,7 +107,7 @@ class BlockingTaskClient(object): :Returns: A `TaskResult` object that encapsulates the task result. """ - return blockingCallFromThread(self.task_controller.get_task_result, + return self._bcft(self.task_controller.get_task_result, taskid, block) def abort(self, taskid): @@ -100,7 +118,7 @@ class BlockingTaskClient(object): taskid : int The taskid of the task to be aborted. """ - return blockingCallFromThread(self.task_controller.abort, taskid) + return self._bcft(self.task_controller.abort, taskid) def barrier(self, taskids): """Block until a set of tasks are completed. @@ -109,7 +127,7 @@ class BlockingTaskClient(object): taskids : list, tuple A sequence of taskids to block on. """ - return blockingCallFromThread(self.task_controller.barrier, taskids) + return self._bcft(self.task_controller.barrier, taskids) def spin(self): """ @@ -118,7 +136,7 @@ class BlockingTaskClient(object): This method only needs to be called in unusual situations where the scheduler is idle for some reason. """ - return blockingCallFromThread(self.task_controller.spin) + return self._bcft(self.task_controller.spin) def queue_status(self, verbose=False): """ @@ -132,7 +150,7 @@ class BlockingTaskClient(object): :Returns: A dict with the queue status. """ - return blockingCallFromThread(self.task_controller.queue_status, verbose) + return self._bcft(self.task_controller.queue_status, verbose) def clear(self): """ @@ -143,7 +161,7 @@ class BlockingTaskClient(object): tasks. Users should call this periodically to clean out these cached task results. """ - return blockingCallFromThread(self.task_controller.clear) + return self._bcft(self.task_controller.clear) def map(self, func, *sequences): """ diff --git a/IPython/kernel/taskfc.py b/IPython/kernel/taskfc.py index c559f64..4b2c47c 100644 --- a/IPython/kernel/taskfc.py +++ b/IPython/kernel/taskfc.py @@ -19,17 +19,17 @@ __docformat__ = "restructuredtext en" #------------------------------------------------------------------------------- import cPickle as pickle -import xmlrpclib, copy from zope.interface import Interface, implements from twisted.internet import defer -from twisted.python import components, failure +from twisted.python import components -from foolscap import Referenceable +try: + from foolscap.api import Referenceable +except ImportError: + from foolscap import Referenceable -from IPython.kernel.twistedutil import blockingCallFromThread -from IPython.kernel import error, task as taskmodule, taskclient -from IPython.kernel.pickleutil import can, uncan +from IPython.kernel import task as taskmodule from IPython.kernel.clientinterfaces import ( IFCClientInterfaceProvider, IBlockingClientAdaptor diff --git a/IPython/kernel/tests/test_contexts.py b/IPython/kernel/tests/test_contexts.py deleted file mode 100644 index fe726c3..0000000 --- a/IPython/kernel/tests/test_contexts.py +++ /dev/null @@ -1,46 +0,0 @@ -# Tell nose to skip this module -__test__ = {} - -#from __future__ import with_statement - -# XXX This file is currently disabled to preserve 2.4 compatibility. - -#def test_simple(): -if 0: - - # XXX - for now, we need a running cluster to be started separately. The - # daemon work is almost finished, and will make much of this unnecessary. - from IPython.kernel import client - mec = client.MultiEngineClient(('127.0.0.1',10105)) - - try: - mec.get_ids() - except ConnectionRefusedError: - import os, time - os.system('ipcluster -n 2 &') - time.sleep(2) - mec = client.MultiEngineClient(('127.0.0.1',10105)) - - mec.block = False - - import itertools - c = itertools.count() - - parallel = RemoteMultiEngine(mec) - - mec.pushAll() - - ## with parallel as pr: - ## # A comment - ## remote() # this means the code below only runs remotely - ## print 'Hello remote world' - ## x = range(10) - ## # Comments are OK - ## # Even misindented. - ## y = x+1 - - - ## with pfor('i',sequence) as pr: - ## print x[i] - - print pr.x + pr.y diff --git a/IPython/kernel/tests/test_enginefc.py b/IPython/kernel/tests/test_enginefc.py index 502b421..1110652 100644 --- a/IPython/kernel/tests/test_enginefc.py +++ b/IPython/kernel/tests/test_enginefc.py @@ -53,8 +53,8 @@ class EngineFCTest(DeferredTestCase, # Start a server and append to self.servers self.controller_reference = FCRemoteEngineRefFromService(self) self.controller_tub = Tub() - self.controller_tub.listenOn('tcp:10105:interface=127.0.0.1') - self.controller_tub.setLocation('127.0.0.1:10105') + self.controller_tub.listenOn('tcp:10111:interface=127.0.0.1') + self.controller_tub.setLocation('127.0.0.1:10111') furl = self.controller_tub.registerReference(self.controller_reference) self.controller_tub.startService() diff --git a/IPython/kernel/tests/test_multienginefc.py b/IPython/kernel/tests/test_multienginefc.py index 340c617..205f320 100644 --- a/IPython/kernel/tests/test_multienginefc.py +++ b/IPython/kernel/tests/test_multienginefc.py @@ -27,7 +27,7 @@ from IPython.kernel.multiengine import IMultiEngine from IPython.kernel.tests.multienginetest import IFullSynchronousMultiEngineTestCase from IPython.kernel.multienginefc import IFCSynchronousMultiEngine from IPython.kernel import multiengine as me -from IPython.kernel.clientconnector import ClientConnector +from IPython.kernel.clientconnector import AsyncClientConnector from IPython.kernel.parallelfunction import ParallelFunction from IPython.kernel.error import CompositeError from IPython.kernel.util import printer @@ -40,37 +40,30 @@ def _raise_it(f): e.raise_exception() -class FullSynchronousMultiEngineTestCase(DeferredTestCase, IFullSynchronousMultiEngineTestCase): +class FullSynchronousMultiEngineTestCase( + DeferredTestCase, IFullSynchronousMultiEngineTestCase): - # XXX (fperez) this is awful: I'm fully disabling this entire test class. - # Right now it's blocking the tests from running at all, and I don't know - # how to fix it. I hope Brian can have a stab at it, but at least by doing - # this we can run the entire suite to completion. - # Once the problem is cleared, remove this skip method. - skip = True - # END XXX - def setUp(self): - + self.engines = [] - + self.controller = ControllerService() self.controller.startService() self.imultiengine = IMultiEngine(self.controller) self.mec_referenceable = IFCSynchronousMultiEngine(self.imultiengine) self.controller_tub = Tub() - self.controller_tub.listenOn('tcp:10105:interface=127.0.0.1') - self.controller_tub.setLocation('127.0.0.1:10105') - + self.controller_tub.listenOn('tcp:10111:interface=127.0.0.1') + self.controller_tub.setLocation('127.0.0.1:10111') + furl = self.controller_tub.registerReference(self.mec_referenceable) self.controller_tub.startService() - - self.client_tub = ClientConnector() - d = self.client_tub.get_multiengine_client(furl) + + self.client_tub = AsyncClientConnector() + d = self.client_tub.get_multiengine_client(furl_or_file=furl) d.addCallback(self.handle_got_client) return d - + def handle_got_client(self, client): self.multiengine = client @@ -95,7 +88,7 @@ class FullSynchronousMultiEngineTestCase(DeferredTestCase, IFullSynchronousMulti self.assertEquals(m.dist,'b') self.assertEquals(m.targets,'all') self.assertEquals(m.block,True) - + def test_map_default(self): self.addEngine(4) m = self.multiengine.mapper() @@ -104,7 +97,7 @@ class FullSynchronousMultiEngineTestCase(DeferredTestCase, IFullSynchronousMulti d.addCallback(lambda _: self.multiengine.map(lambda x: 2*x, range(10))) d.addCallback(lambda r: self.assertEquals(r,[2*x for x in range(10)])) return d - + def test_map_noblock(self): self.addEngine(4) m = self.multiengine.mapper(block=False) @@ -112,14 +105,14 @@ class FullSynchronousMultiEngineTestCase(DeferredTestCase, IFullSynchronousMulti d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True)) d.addCallback(lambda r: self.assertEquals(r,[2*x for x in range(10)])) return d - + def test_mapper_fail(self): self.addEngine(4) m = self.multiengine.mapper() d = m.map(lambda x: 1/0, range(10)) d.addBoth(lambda f: self.assertRaises(ZeroDivisionError, _raise_it, f)) return d - + def test_parallel(self): self.addEngine(4) p = self.multiengine.parallel() @@ -129,7 +122,7 @@ class FullSynchronousMultiEngineTestCase(DeferredTestCase, IFullSynchronousMulti d = f(range(10)) d.addCallback(lambda r: self.assertEquals(r,[2*x for x in range(10)])) return d - + def test_parallel_noblock(self): self.addEngine(1) p = self.multiengine.parallel(block=False) @@ -140,7 +133,7 @@ class FullSynchronousMultiEngineTestCase(DeferredTestCase, IFullSynchronousMulti d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True)) d.addCallback(lambda r: self.assertEquals(r,[2*x for x in range(10)])) return d - + def test_parallel_fail(self): self.addEngine(4) p = self.multiengine.parallel() @@ -150,3 +143,4 @@ class FullSynchronousMultiEngineTestCase(DeferredTestCase, IFullSynchronousMulti d = f(range(10)) d.addBoth(lambda f: self.assertRaises(ZeroDivisionError, _raise_it, f)) return d + diff --git a/IPython/kernel/tests/test_taskfc.py b/IPython/kernel/tests/test_taskfc.py index 589cf3e..79958c3 100644 --- a/IPython/kernel/tests/test_taskfc.py +++ b/IPython/kernel/tests/test_taskfc.py @@ -31,7 +31,7 @@ from IPython.kernel.multienginefc import IFCSynchronousMultiEngine from IPython.kernel.taskfc import IFCTaskController from IPython.kernel.util import printer from IPython.kernel.tests.tasktest import ITaskControllerTestCase -from IPython.kernel.clientconnector import ClientConnector +from IPython.kernel.clientconnector import AsyncClientConnector from IPython.kernel.error import CompositeError from IPython.kernel.parallelfunction import ParallelFunction @@ -48,42 +48,34 @@ def _raise_it(f): class TaskTest(DeferredTestCase, ITaskControllerTestCase): - # XXX (fperez) this is awful: I'm fully disabling this entire test class. - # Right now it's blocking the tests from running at all, and I don't know - # how to fix it. I hope Brian can have a stab at it, but at least by doing - # this we can run the entire suite to completion. - # Once the problem is cleared, remove this skip method. - skip = True - # END XXX - def setUp(self): - + self.engines = [] - + self.controller = cs.ControllerService() self.controller.startService() self.imultiengine = me.IMultiEngine(self.controller) self.itc = taskmodule.ITaskController(self.controller) self.itc.failurePenalty = 0 - + self.mec_referenceable = IFCSynchronousMultiEngine(self.imultiengine) self.tc_referenceable = IFCTaskController(self.itc) - + self.controller_tub = Tub() - self.controller_tub.listenOn('tcp:10105:interface=127.0.0.1') - self.controller_tub.setLocation('127.0.0.1:10105') - + self.controller_tub.listenOn('tcp:10111:interface=127.0.0.1') + self.controller_tub.setLocation('127.0.0.1:10111') + mec_furl = self.controller_tub.registerReference(self.mec_referenceable) tc_furl = self.controller_tub.registerReference(self.tc_referenceable) self.controller_tub.startService() - - self.client_tub = ClientConnector() - d = self.client_tub.get_multiengine_client(mec_furl) + + self.client_tub = AsyncClientConnector() + d = self.client_tub.get_multiengine_client(furl_or_file=mec_furl) d.addCallback(self.handle_mec_client) - d.addCallback(lambda _: self.client_tub.get_task_client(tc_furl)) + d.addCallback(lambda _: self.client_tub.get_task_client(furl_or_file=tc_furl)) d.addCallback(self.handle_tc_client) return d - + def handle_mec_client(self, client): self.multiengine = client @@ -103,7 +95,7 @@ class TaskTest(DeferredTestCase, ITaskControllerTestCase): d.addBoth(lambda _: self.controller.stopService()) dlist.append(d) return defer.DeferredList(dlist) - + def test_mapper(self): self.addEngine(1) m = self.tc.mapper() @@ -114,7 +106,7 @@ class TaskTest(DeferredTestCase, ITaskControllerTestCase): self.assertEquals(m.recovery_task,None) self.assertEquals(m.depend,None) self.assertEquals(m.block,True) - + def test_map_default(self): self.addEngine(1) m = self.tc.mapper() @@ -123,21 +115,21 @@ class TaskTest(DeferredTestCase, ITaskControllerTestCase): d.addCallback(lambda _: self.tc.map(lambda x: 2*x, range(10))) d.addCallback(lambda r: self.assertEquals(r,[2*x for x in range(10)])) return d - + def test_map_noblock(self): self.addEngine(1) m = self.tc.mapper(block=False) d = m.map(lambda x: 2*x, range(10)) d.addCallback(lambda r: self.assertEquals(r,[x for x in range(10)])) return d - + def test_mapper_fail(self): self.addEngine(1) m = self.tc.mapper() d = m.map(lambda x: 1/0, range(10)) d.addBoth(lambda f: self.assertRaises(ZeroDivisionError, _raise_it, f)) return d - + def test_parallel(self): self.addEngine(1) p = self.tc.parallel() @@ -147,7 +139,7 @@ class TaskTest(DeferredTestCase, ITaskControllerTestCase): d = f(range(10)) d.addCallback(lambda r: self.assertEquals(r,[2*x for x in range(10)])) return d - + def test_parallel_noblock(self): self.addEngine(1) p = self.tc.parallel(block=False) @@ -157,7 +149,7 @@ class TaskTest(DeferredTestCase, ITaskControllerTestCase): d = f(range(10)) d.addCallback(lambda r: self.assertEquals(r,[x for x in range(10)])) return d - + def test_parallel_fail(self): self.addEngine(1) p = self.tc.parallel() @@ -167,3 +159,4 @@ class TaskTest(DeferredTestCase, ITaskControllerTestCase): d = f(range(10)) d.addBoth(lambda f: self.assertRaises(ZeroDivisionError, _raise_it, f)) return d + diff --git a/IPython/kernel/twistedutil.py b/IPython/kernel/twistedutil.py index 712a83b..ff73438 100644 --- a/IPython/kernel/twistedutil.py +++ b/IPython/kernel/twistedutil.py @@ -15,7 +15,7 @@ #----------------------------------------------------------------------------- import os, sys -import threading, Queue, atexit +import threading, Queue import twisted from twisted.internet import defer, reactor diff --git a/IPython/kernel/winhpcjob.py b/IPython/kernel/winhpcjob.py index de1680d..c7dc1c9 100644 --- a/IPython/kernel/winhpcjob.py +++ b/IPython/kernel/winhpcjob.py @@ -23,12 +23,10 @@ import re import uuid from xml.etree import ElementTree as ET -from xml.dom import minidom from IPython.core.component import Component -from IPython.external import Itpl from IPython.utils.traitlets import ( - Str, Int, List, Unicode, Instance, + Str, Int, List, Instance, Enum, Bool, CStr ) diff --git a/IPython/lib/backgroundjobs.py b/IPython/lib/backgroundjobs.py index 6f49522..aec4526 100644 --- a/IPython/lib/backgroundjobs.py +++ b/IPython/lib/backgroundjobs.py @@ -31,7 +31,7 @@ import sys import threading from IPython.core.ultratb import AutoFormattedTB -from IPython.utils.genutils import warn,error +from IPython.utils.warn import warn, error class BackgroundJobManager: """Class to manage a pool of backgrounded threaded jobs. diff --git a/IPython/lib/deepreload.py b/IPython/lib/deepreload.py index f2ea5f7..5925ee0 100644 --- a/IPython/lib/deepreload.py +++ b/IPython/lib/deepreload.py @@ -1,15 +1,18 @@ # -*- coding: utf-8 -*- """ A module to change reload() so that it acts recursively. -To enable it type: - >>> import __builtin__, deepreload - >>> __builtin__.reload = deepreload.reload +To enable it type:: -You can then disable it with: - >>> __builtin__.reload = deepreload.original_reload + import __builtin__, deepreload + __builtin__.reload = deepreload.reload + +You can then disable it with:: + + __builtin__.reload = deepreload.original_reload -Alternatively, you can add a dreload builtin alongside normal reload with: - >>> __builtin__.dreload = deepreload.reload +Alternatively, you can add a dreload builtin alongside normal reload with:: + + __builtin__.dreload = deepreload.reload This code is almost entirely based on knee.py from the standard library. """ diff --git a/IPython/lib/demo.py b/IPython/lib/demo.py index d5989e6..53a3ea0 100644 --- a/IPython/lib/demo.py +++ b/IPython/lib/demo.py @@ -176,7 +176,8 @@ import shlex import sys from IPython.utils.PyColorize import Parser -from IPython.utils.genutils import marquee, file_read, file_readlines, Term +from IPython.utils.io import file_read, file_readlines, Term +from IPython.utils.text import marquee __all__ = ['Demo','IPythonDemo','LineDemo','IPythonLineDemo','DemoError'] @@ -543,7 +544,7 @@ class ClearMixin(object): """Method called before executing each block. This one simply clears the screen.""" - from IPython.utils.platutils import term_clear + from IPython.utils.terminal import term_clear term_clear() class ClearDemo(ClearMixin,Demo): diff --git a/IPython/core/pylabtools.py b/IPython/lib/pylabtools.py similarity index 99% rename from IPython/core/pylabtools.py rename to IPython/lib/pylabtools.py index f56131a..73a27cf 100644 --- a/IPython/core/pylabtools.py +++ b/IPython/lib/pylabtools.py @@ -12,10 +12,12 @@ Fernando Perez. # Distributed under the terms of the BSD License. The full license is in # the file COPYING, distributed as part of this software. #----------------------------------------------------------------------------- + #----------------------------------------------------------------------------- # Imports #----------------------------------------------------------------------------- -from IPython.utils.genutils import flag_calls + +from IPython.utils.decorators import flag_calls #----------------------------------------------------------------------------- # Main classes and functions diff --git a/IPython/quarantine/InterpreterExec.py b/IPython/quarantine/InterpreterExec.py index 70bebac..9faba79 100644 --- a/IPython/quarantine/InterpreterExec.py +++ b/IPython/quarantine/InterpreterExec.py @@ -50,7 +50,7 @@ del InteractiveShell,prefilter_shell # Provide pysh and further shell-oriented services import os,sys,shutil -from IPython.utils.genutils import system,shell,getoutput,getoutputerror +from IPython.utils.process import system,shell,getoutput,getoutputerror # Short aliases for getting shell output as a string and a list sout = getoutput diff --git a/IPython/quarantine/ext_rescapture.py b/IPython/quarantine/ext_rescapture.py index 6f5731a..eb0c391 100644 --- a/IPython/quarantine/ext_rescapture.py +++ b/IPython/quarantine/ext_rescapture.py @@ -10,6 +10,7 @@ var = !ls from IPython.core import ipapi from IPython.core.error import TryNext +from IPython.utils.text import make_quoted_expr from IPython.utils.genutils import * ip = ipapi.get() diff --git a/IPython/quarantine/ipy_greedycompleter.py b/IPython/quarantine/ipy_greedycompleter.py index f928e45..1804338 100644 --- a/IPython/quarantine/ipy_greedycompleter.py +++ b/IPython/quarantine/ipy_greedycompleter.py @@ -12,7 +12,7 @@ do the same in default completer. from IPython.core import ipapi from IPython.core.error import TryNext from IPython.utils import generics -from IPython.utils.genutils import dir2 +from IPython.utils.dir2 import dir2 def attr_matches(self, text): """Compute matches when text contains a dot. diff --git a/IPython/quarantine/ipy_jot.py b/IPython/quarantine/ipy_jot.py index 5b2e65e..16fd4f2 100644 --- a/IPython/quarantine/ipy_jot.py +++ b/IPython/quarantine/ipy_jot.py @@ -16,6 +16,7 @@ import pickleshare import inspect,pickle,os,sys,textwrap from IPython.core.fakemodule import FakeModule from IPython.utils.ipstruct import Struct +from IPython.utils.warn import error def refresh_variables(ip, key=None): diff --git a/IPython/quarantine/ipy_pydb.py b/IPython/quarantine/ipy_pydb.py index 278a336..67b45f0 100644 --- a/IPython/quarantine/ipy_pydb.py +++ b/IPython/quarantine/ipy_pydb.py @@ -1,6 +1,6 @@ import inspect from IPython.core import ipapi -from IPython.utils.genutils import arg_split +from IPython.utils.process import arg_split ip = ipapi.get() from IPython.core import debugger diff --git a/IPython/quarantine/jobctrl.py b/IPython/quarantine/jobctrl.py index 9a38ce5..7518a52 100644 --- a/IPython/quarantine/jobctrl.py +++ b/IPython/quarantine/jobctrl.py @@ -45,10 +45,9 @@ from subprocess import * import os,shlex,sys,time import threading,Queue -from IPython.utils import genutils - from IPython.core import ipapi from IPython.core.error import TryNext +from IPython.utils.text import make_quoted_expr if os.name == 'nt': def kill_process(pid): @@ -126,8 +125,8 @@ def jobctrl_prefilter_f(self,line): line = ip.expand_aliases(fn,rest) if not _jobq: - return 'get_ipython().startjob(%s)' % genutils.make_quoted_expr(line) - return 'get_ipython().jobq(%s)' % genutils.make_quoted_expr(line) + return 'get_ipython().startjob(%s)' % make_quoted_expr(line) + return 'get_ipython().jobq(%s)' % make_quoted_expr(line) raise TryNext diff --git a/IPython/testing/__init__.py b/IPython/testing/__init__.py index 830585c..ea70b45 100644 --- a/IPython/testing/__init__.py +++ b/IPython/testing/__init__.py @@ -1,6 +1,17 @@ """Testing support (tools to test IPython itself). """ +#----------------------------------------------------------------------------- +# Copyright (C) 2009 The IPython Development Team +# +# Distributed under the terms of the BSD License. The full license is in +# the file COPYING, distributed as part of this software. +#----------------------------------------------------------------------------- + +#----------------------------------------------------------------------------- +# Functions +#----------------------------------------------------------------------------- + # User-level entry point for testing def test(): """Run the entire IPython test suite. diff --git a/IPython/testing/_doctest26.py b/IPython/testing/_doctest26.py index b3b3e5f..e30c024 100644 --- a/IPython/testing/_doctest26.py +++ b/IPython/testing/_doctest26.py @@ -4,6 +4,17 @@ This is just so we can use 2.6 features when running in 2.5, the code below is copied verbatim from the stdlib's collections and doctest modules. """ +#----------------------------------------------------------------------------- +# Copyright (C) 2009 The IPython Development Team +# +# Distributed under the terms of the BSD License. The full license is in +# the file COPYING, distributed as part of this software. +#----------------------------------------------------------------------------- + +#----------------------------------------------------------------------------- +# Imports +#----------------------------------------------------------------------------- + from keyword import iskeyword as _iskeyword from operator import itemgetter as _itemgetter import sys as _sys diff --git a/IPython/testing/_paramtestpy2.py b/IPython/testing/_paramtestpy2.py index 07e3a9a..d9a05d9 100644 --- a/IPython/testing/_paramtestpy2.py +++ b/IPython/testing/_paramtestpy2.py @@ -1,10 +1,17 @@ """Implementation of the parametric test support for Python 2.x """ + +#----------------------------------------------------------------------------- +# Copyright (C) 2009 The IPython Development Team +# +# Distributed under the terms of the BSD License. The full license is in +# the file COPYING, distributed as part of this software. +#----------------------------------------------------------------------------- + #----------------------------------------------------------------------------- # Imports #----------------------------------------------------------------------------- -# Stdlib import unittest from compiler.consts import CO_GENERATOR diff --git a/IPython/testing/_paramtestpy3.py b/IPython/testing/_paramtestpy3.py index 0e16a36..200ca0d 100644 --- a/IPython/testing/_paramtestpy3.py +++ b/IPython/testing/_paramtestpy3.py @@ -3,11 +3,18 @@ Thanks for the py3 version to Robert Collins, from the Testing in Python mailing list. """ + +#----------------------------------------------------------------------------- +# Copyright (C) 2009 The IPython Development Team +# +# Distributed under the terms of the BSD License. The full license is in +# the file COPYING, distributed as part of this software. +#----------------------------------------------------------------------------- + #----------------------------------------------------------------------------- # Imports #----------------------------------------------------------------------------- -# Stdlib import unittest from unittest import TestSuite diff --git a/IPython/testing/decorators_trial.py b/IPython/testing/decorators_trial.py new file mode 100644 index 0000000..231054b --- /dev/null +++ b/IPython/testing/decorators_trial.py @@ -0,0 +1,132 @@ +# encoding: utf-8 +""" +Testing related decorators for use with twisted.trial. + +The decorators in this files are designed to follow the same API as those +in the decorators module (in this same directory). But they can be used +with twisted.trial +""" + +#----------------------------------------------------------------------------- +# Copyright (C) 2008-2009 The IPython Development Team +# +# Distributed under the terms of the BSD License. The full license is in +# the file COPYING, distributed as part of this software. +#----------------------------------------------------------------------------- + +#----------------------------------------------------------------------------- +# Imports +#----------------------------------------------------------------------------- + +import os +import sys + +from IPython.testing.decorators import make_label_dec + +#----------------------------------------------------------------------------- +# Testing decorators +#----------------------------------------------------------------------------- + + +def skipif(skip_condition, msg=None): + """Create a decorator that marks a test function for skipping. + + The is a decorator factory that returns a decorator that will + conditionally skip a test based on the value of skip_condition. The + skip_condition argument can either be a boolean or a callable that returns + a boolean. + + Parameters + ---------- + skip_condition : boolean or callable + If this evaluates to True, the test is skipped. + msg : str + The message to print if the test is skipped. + + Returns + ------- + decorator : function + The decorator function that can be applied to the test function. + """ + + def skip_decorator(f): + + # Allow for both boolean or callable skip conditions. + if callable(skip_condition): + skip_val = lambda : skip_condition() + else: + skip_val = lambda : skip_condition + + if msg is None: + out = 'Test skipped due to test condition.' + else: + out = msg + final_msg = "Skipping test: %s. %s" % (f.__name__,out) + + if skip_val(): + f.skip = final_msg + + return f + return skip_decorator + + +def skip(msg=None): + """Create a decorator that marks a test function for skipping. + + This is a decorator factory that returns a decorator that will cause + tests to be skipped. + + Parameters + ---------- + msg : str + Optional message to be added. + + Returns + ------- + decorator : function + Decorator, which, when applied to a function, sets the skip + attribute of the function causing `twisted.trial` to skip it. + """ + + return skipif(True,msg) + + +def numpy_not_available(): + """Can numpy be imported? Returns true if numpy does NOT import. + + This is used to make a decorator to skip tests that require numpy to be + available, but delay the 'import numpy' to test execution time. + """ + try: + import numpy + np_not_avail = False + except ImportError: + np_not_avail = True + + return np_not_avail + +#----------------------------------------------------------------------------- +# Decorators for public use +#----------------------------------------------------------------------------- + +# Decorators to skip certain tests on specific platforms. +skip_win32 = skipif(sys.platform == 'win32', + "This test does not run under Windows") +skip_linux = skipif(sys.platform == 'linux2', + "This test does not run under Linux") +skip_osx = skipif(sys.platform == 'darwin',"This test does not run under OS X") + +# Decorators to skip tests if not on specific platforms. +skip_if_not_win32 = skipif(sys.platform != 'win32', + "This test only runs under Windows") +skip_if_not_linux = skipif(sys.platform != 'linux2', + "This test only runs under Linux") +skip_if_not_osx = skipif(sys.platform != 'darwin', + "This test only runs under OSX") + +# Other skip decorators +skipif_not_numpy = skipif(numpy_not_available,"This test requires numpy") + +skipknownfailure = skip('This test is known to fail') + + diff --git a/IPython/testing/globalipapp.py b/IPython/testing/globalipapp.py index cd3c235..714554e 100644 --- a/IPython/testing/globalipapp.py +++ b/IPython/testing/globalipapp.py @@ -9,18 +9,22 @@ done. from __future__ import absolute_import #----------------------------------------------------------------------------- -# Module imports +# Copyright (C) 2009 The IPython Development Team +# +# Distributed under the terms of the BSD License. The full license is in +# the file COPYING, distributed as part of this software. +#----------------------------------------------------------------------------- + +#----------------------------------------------------------------------------- +# Imports #----------------------------------------------------------------------------- -# From the standard library import __builtin__ import commands -import new import os import sys from . import tools -from IPython.utils.genutils import Term #----------------------------------------------------------------------------- # Functions @@ -35,7 +39,7 @@ class py_file_finder(object): self.test_filename = test_filename def __call__(self,name): - from IPython.utils.genutils import get_py_filename + from IPython.utils.path import get_py_filename try: return get_py_filename(name) except IOError: @@ -93,24 +97,25 @@ class ipnsdict(dict): # correct for that ourselves, to ensure consitency with the 'real' # ipython. self['__builtins__'] = __builtin__ - + def get_ipython(): # This will get replaced by the real thing once we start IPython below return start_ipython() + def start_ipython(): """Start a global IPython shell, which we need for IPython-specific syntax. """ global get_ipython # This function should only ever run once! - if hasattr(start_ipython,'already_called'): + if hasattr(start_ipython, 'already_called'): return start_ipython.already_called = True # Ok, first time we're called, go ahead - from IPython.core import ipapp, iplib + from IPython.core import iplib def xsys(cmd): """Execute a command and print its output. @@ -128,26 +133,27 @@ def start_ipython(): _main = sys.modules.get('__main__') # Create custom argv and namespaces for our IPython to be test-friendly - argv = tools.default_argv() - user_ns, global_ns = iplib.make_user_namespaces(ipnsdict(), {}) - + config = tools.default_config() + # Create and initialize our test-friendly IPython instance. - ip = ipapp.IPythonApp(argv, user_ns=user_ns, user_global_ns=global_ns) - ip.initialize() + shell = iplib.InteractiveShell( + parent=None, config=config, + user_ns=ipnsdict(), user_global_ns={} + ) # A few more tweaks needed for playing nicely with doctests... # These traps are normally only active for interactive use, set them # permanently since we'll be mocking interactive sessions. - ip.shell.builtin_trap.set() + shell.builtin_trap.set() # Set error printing to stdout so nose can doctest exceptions - ip.shell.InteractiveTB.out_stream = 'stdout' + shell.InteractiveTB.out_stream = 'stdout' # Modify the IPython system call with one that uses getoutput, so that we # can capture subcommands and print them to Python's stdout, otherwise the # doctest machinery would miss them. - ip.shell.system = xsys + shell.system = xsys # IPython is ready, now clean up some global state... @@ -160,7 +166,7 @@ def start_ipython(): # So that ipython magics and aliases can be doctested (they work by making # a call into a global _ip object). Also make the top-level get_ipython # now return this without recursively calling here again. - _ip = ip.shell + _ip = shell get_ipython = _ip.get_ipython __builtin__._ip = _ip __builtin__.get_ipython = get_ipython diff --git a/IPython/testing/iptest.py b/IPython/testing/iptest.py index 946d2c0..2f8453f 100644 --- a/IPython/testing/iptest.py +++ b/IPython/testing/iptest.py @@ -17,13 +17,19 @@ will change in the future. """ #----------------------------------------------------------------------------- -# Module imports +# Copyright (C) 2009 The IPython Development Team +# +# Distributed under the terms of the BSD License. The full license is in +# the file COPYING, distributed as part of this software. +#----------------------------------------------------------------------------- + +#----------------------------------------------------------------------------- +# Imports #----------------------------------------------------------------------------- # Stdlib import os import os.path as path -import platform import signal import sys import subprocess @@ -31,15 +37,6 @@ import tempfile import time import warnings - -# Ugly, but necessary hack to ensure the test suite finds our version of -# IPython and not a possibly different one that may exist system-wide. -# Note that this must be done here, so the imports that come next work -# correctly even if IPython isn't installed yet. -p = os.path -ippath = p.abspath(p.join(p.dirname(__file__),'..','..')) -sys.path.insert(0, ippath) - # Note: monkeypatch! # We need to monkeypatch a small problem in nose itself first, before importing # it for actual use. This should get into nose upstream, but its release cycle @@ -50,11 +47,11 @@ import nose.plugins.builtin from nose.core import TestProgram # Our own imports -from IPython.core import release -from IPython.utils import genutils -from IPython.utils.platutils import find_cmd, FindCmdError +from IPython.utils.path import get_ipython_module_path +from IPython.utils.process import find_cmd, pycmd2argv +from IPython.utils.sysinfo import sys_info + from IPython.testing import globalipapp -from IPython.testing import tools from IPython.testing.plugin.ipdoctest import IPythonDoctest pjoin = path.join @@ -64,16 +61,11 @@ pjoin = path.join # Globals #----------------------------------------------------------------------------- -# By default, we assume IPython has been installed. But if the test suite is -# being run from a source tree that has NOT been installed yet, this flag can -# be set to False by the entry point scripts, to let us know that we must call -# the source tree versions of the scripts which manipulate sys.path instead of -# assuming that things exist system-wide. -INSTALLED = True #----------------------------------------------------------------------------- # Warnings control #----------------------------------------------------------------------------- + # Twisted generates annoying warnings with Python 2.6, as will do other code # that imports 'sets' as of today warnings.filterwarnings('ignore', 'the sets module is deprecated', @@ -124,9 +116,7 @@ have['gobject'] = test_for('gobject') def report(): """Return a string with a summary report of test-related variables.""" - out = [ genutils.sys_info() ] - - out.append('\nRunning from an installed IPython: %s\n' % INSTALLED) + out = [ sys_info() ] avail = [] not_avail = [] @@ -152,11 +142,11 @@ def report(): def make_exclude(): """Make patterns of modules and packages to exclude from testing. - + For the IPythonDoctest plugin, we need to exclude certain patterns that cause testing problems. We should strive to minimize the number of - skipped modules, since this means untested code. As the testing - machinery solidifies, this list should eventually become empty. + skipped modules, since this means untested code. + These modules and packages will NOT get scanned by nose at all for tests. """ # Simple utility to make IPython paths more readably, we need a lot of @@ -198,18 +188,12 @@ def make_exclude(): if not have['objc']: exclusions.append(ipjoin('frontend', 'cocoa')) - if not sys.platform == 'win32': - exclusions.append(ipjoin('utils', 'platutils_win32')) - # These have to be skipped on win32 because the use echo, rm, cd, etc. # See ticket https://bugs.launchpad.net/bugs/366982 if sys.platform == 'win32': exclusions.append(ipjoin('testing', 'plugin', 'test_exampleip')) exclusions.append(ipjoin('testing', 'plugin', 'dtexample')) - if not os.name == 'posix': - exclusions.append(ipjoin('utils', 'platutils_posix')) - if not have['pexpect']: exclusions.extend([ipjoin('scripts', 'irunner'), ipjoin('lib', 'irunner')]) @@ -255,20 +239,13 @@ class IPTester(object): """Create new test runner.""" p = os.path if runner == 'iptest': - if INSTALLED: - self.runner = tools.cmd2argv( - p.abspath(find_cmd('iptest'))) + sys.argv[1:] - else: - # Find our own 'iptest' script OS-level entry point. Don't - # look system-wide, so we are sure we pick up *this one*. And - # pass through to subprocess call our own sys.argv - ippath = p.abspath(p.join(p.dirname(__file__),'..','..')) - script = p.join(ippath, 'iptest.py') - self.runner = tools.cmd2argv(script) + sys.argv[1:] - - else: + iptest_app = get_ipython_module_path('IPython.testing.iptest') + self.runner = pycmd2argv(iptest_app) + sys.argv[1:] + elif runner == 'trial': # For trial, it needs to be installed system-wide - self.runner = tools.cmd2argv(p.abspath(find_cmd('trial'))) + self.runner = pycmd2argv(p.abspath(find_cmd('trial'))) + else: + raise Exception('Not a valid test runner: %s' % repr(runner)) if params is None: params = [] if isinstance(params, str): @@ -290,6 +267,8 @@ class IPTester(object): # fashioned' way to do it, but it works just fine. If someone # later can clean this up that's fine, as long as the tests run # reliably in win32. + # What types of problems are you having. They may be related to + # running Python in unboffered mode. BG. return os.system(' '.join(self.call_args)) else: def _run_cmd(self): @@ -343,9 +322,9 @@ def make_runners(): # And add twisted ones if conditions are met if have['zope.interface'] and have['twisted'] and have['foolscap']: - # Note that we list the kernel here, though the bulk of it is - # twisted-based, because nose picks up doctests that twisted doesn't. - nose_pkg_names.append('kernel') + # We only list IPython.kernel for testing using twisted.trial as + # nose and twisted.trial have conflicts that make the testing system + # unstable. trial_pkg_names.append('kernel') # For debugging this code, only load quick stuff diff --git a/IPython/testing/ipunittest.py b/IPython/testing/ipunittest.py index d4a7609..e593c74 100644 --- a/IPython/testing/ipunittest.py +++ b/IPython/testing/ipunittest.py @@ -31,7 +31,6 @@ from __future__ import absolute_import # the file COPYING, distributed as part of this software. #----------------------------------------------------------------------------- - #----------------------------------------------------------------------------- # Imports #----------------------------------------------------------------------------- diff --git a/IPython/testing/mkdoctests.py b/IPython/testing/mkdoctests.py index 4ea9335..bfec99e 100644 --- a/IPython/testing/mkdoctests.py +++ b/IPython/testing/mkdoctests.py @@ -38,7 +38,7 @@ import tempfile # IPython-specific libraries from IPython.lib import irunner -from IPython.utils.genutils import fatal +from IPython.utils.warn import fatal class IndentOut(object): """A simple output stream that indents all output by a fixed amount. diff --git a/IPython/testing/nosepatch.py b/IPython/testing/nosepatch.py index 1065a3e..2458e39 100644 --- a/IPython/testing/nosepatch.py +++ b/IPython/testing/nosepatch.py @@ -5,10 +5,25 @@ Once this is fixed in upstream nose we can disable it. Note: merely importing this module causes the monkeypatch to be applied.""" +#----------------------------------------------------------------------------- +# Copyright (C) 2009 The IPython Development Team +# +# Distributed under the terms of the BSD License. The full license is in +# the file COPYING, distributed as part of this software. +#----------------------------------------------------------------------------- + +#----------------------------------------------------------------------------- +# Imports +#----------------------------------------------------------------------------- + import unittest import nose.loader from inspect import ismethod, isfunction +#----------------------------------------------------------------------------- +# Classes and functions +#----------------------------------------------------------------------------- + def getTestCaseNames(self, testCaseClass): """Override to select with selector, unless config.getTestCaseNamesCompat is True diff --git a/IPython/testing/parametric.py b/IPython/testing/parametric.py index b1bf243..fe8a52a 100644 --- a/IPython/testing/parametric.py +++ b/IPython/testing/parametric.py @@ -5,10 +5,27 @@ parametric code. We just need to double-check that the new code doesn't clash with Twisted (we know it works with nose and unittest). """ -__all__ = ['parametric','Parametric'] +#----------------------------------------------------------------------------- +# Copyright (C) 2009 The IPython Development Team +# +# Distributed under the terms of the BSD License. The full license is in +# the file COPYING, distributed as part of this software. +#----------------------------------------------------------------------------- + +#----------------------------------------------------------------------------- +# Imports +#----------------------------------------------------------------------------- + from twisted.trial.unittest import TestCase +#----------------------------------------------------------------------------- +# Classes and functions +#----------------------------------------------------------------------------- + +__all__ = ['parametric','Parametric'] + + def partial(f, *partial_args, **partial_kwargs): """Generate a partial class method. @@ -20,6 +37,7 @@ def partial(f, *partial_args, **partial_kwargs): return partial_func + def parametric(f): """Mark f as a parametric test. @@ -27,6 +45,7 @@ def parametric(f): f._parametric = True return classmethod(f) + def Parametric(cls): """Register parametric tests with a class. @@ -56,3 +75,4 @@ def Parametric(cls): # rename test generator so it isn't called again by nose test_gen.im_func.func_name = '__done_' + test_name + diff --git a/IPython/testing/tests/test_decorators_trial.py b/IPython/testing/tests/test_decorators_trial.py index 427bede..b6157dc 100644 --- a/IPython/testing/tests/test_decorators_trial.py +++ b/IPython/testing/tests/test_decorators_trial.py @@ -1,6 +1,6 @@ # encoding: utf-8 """ -Tests for decorators.py compatibility with Twisted.trial +Tests for decorators_trial.py """ #----------------------------------------------------------------------------- @@ -14,24 +14,19 @@ Tests for decorators.py compatibility with Twisted.trial # Imports #----------------------------------------------------------------------------- -# Tell nose to skip this module, since this is for twisted only +# Tell nose to skip this module __test__ = {} import os import sys from twisted.trial import unittest -import IPython.testing.decorators as dec +import IPython.testing.decorators_trial as dec #----------------------------------------------------------------------------- # Tests #----------------------------------------------------------------------------- -# Note: this code is identical to that in test_decorators, but that one uses -# stdlib unittest, not the one from twisted, which we are using here. While -# somewhat redundant, we want to check both with the stdlib and with twisted, -# so the duplication is OK. - class TestDecoratorsTrial(unittest.TestCase): @dec.skip() diff --git a/IPython/testing/tools.py b/IPython/testing/tools.py index ab08cff..a6dc9da 100644 --- a/IPython/testing/tools.py +++ b/IPython/testing/tools.py @@ -15,22 +15,22 @@ Authors - Fernando Perez """ -#***************************************************************************** -# Copyright (C) 2009 The IPython Development Team +from __future__ import absolute_import + +#----------------------------------------------------------------------------- +# Copyright (C) 2009 The IPython Development Team # # Distributed under the terms of the BSD License. The full license is in # the file COPYING, distributed as part of this software. -#***************************************************************************** +#----------------------------------------------------------------------------- #----------------------------------------------------------------------------- -# Required modules and packages +# Imports #----------------------------------------------------------------------------- -from __future__ import absolute_import import os import re import sys -import tempfile try: # These tools are used by parts of the runtime, so we make the nose @@ -41,7 +41,10 @@ try: except ImportError: has_nose = False -from IPython.utils import genutils, platutils +from IPython.config.loader import Config +from IPython.utils.process import find_cmd, getoutputerror +from IPython.utils.text import list_strings +from IPython.utils.io import temp_pyfile from . import decorators as dec @@ -49,13 +52,6 @@ from . import decorators as dec # Globals #----------------------------------------------------------------------------- -# By default, we assume IPython has been installed. But if the test suite is -# being run from a source tree that has NOT been installed yet, this flag can -# be set to False by the entry point scripts, to let us know that we must call -# the source tree versions of the scripts which manipulate sys.path instead of -# assuming that things exist system-wide. -INSTALLED = True - # Make a bunch of nose.tools assert wrappers that can be used in test # generators. This will expose an assert* function for each one in nose.tools. @@ -107,7 +103,7 @@ def full_path(startPath,files): ['/a.txt'] """ - files = genutils.list_strings(files) + files = list_strings(files) base = os.path.split(startPath)[0] return [ os.path.join(base,f) for f in files ] @@ -156,63 +152,6 @@ def parse_test_output(txt): parse_test_output.__test__ = False -def cmd2argv(cmd): - r"""Take the path of a command and return a list (argv-style). - - For a given path ``cmd``, this returns [cmd] if cmd's extension is .exe, - .com or .bat, and ['python', cmd] otherwise. - - This is mostly a Windows utility, to deal with the fact that the scripts in - Windows get wrapped in .exe entry points, so we have to call them - differently. - - Parameters - ---------- - cmd : string - The path of the command. - - Returns - ------- - argv-style list. - - Examples - -------- - In [2]: cmd2argv('/usr/bin/ipython') - Out[2]: ['python', '/usr/bin/ipython'] - - In [3]: cmd2argv(r'C:\Python26\Scripts\ipython.exe') - Out[3]: ['C:\\Python26\\Scripts\\ipython.exe'] - """ - ext = os.path.splitext(cmd)[1] - if ext in ['.exe', '.com', '.bat']: - return [cmd] - else: - return ['python', cmd] - - -def temp_pyfile(src, ext='.py'): - """Make a temporary python file, return filename and filehandle. - - Parameters - ---------- - src : string or list of strings (no need for ending newlines if list) - Source code to be written to the file. - - ext : optional, string - Extension for the generated file. - - Returns - ------- - (filename, open filehandle) - It is the caller's responsibility to close the open file and unlink it. - """ - fname = tempfile.mkstemp(ext)[1] - f = open(fname,'w') - f.write(src) - f.flush() - return fname, f - - def default_argv(): """Return a valid default argv for creating testing instances of ipython""" @@ -220,7 +159,16 @@ def default_argv(): # Other defaults to minimize side effects on stdout '--colors=NoColor', '--no-term-title','--no-banner', '--autocall=0'] - + + +def default_config(): + """Return a config object with good defaults for testing.""" + config = Config() + config.InteractiveShell.colors = 'NoColor' + config.InteractiveShell.term_title = False, + config.InteractiveShell.autocall = 0 + return config + def ipexec(fname, options=None): """Utility to call 'ipython filename'. @@ -252,20 +200,12 @@ def ipexec(fname, options=None): _ip = get_ipython() test_dir = os.path.dirname(__file__) - # Find the ipython script from the package we're using, so that the test - # suite can be run from the source tree without an installed IPython - p = os.path - if INSTALLED: - ipython_cmd = platutils.find_cmd('ipython') - else: - ippath = p.abspath(p.join(p.dirname(__file__),'..','..')) - ipython_script = p.join(ippath, 'ipython.py') - ipython_cmd = 'python "%s"' % ipython_script + ipython_cmd = find_cmd('ipython') # Absolute path for filename - full_fname = p.join(test_dir, fname) + full_fname = os.path.join(test_dir, fname) full_cmd = '%s %s %s' % (ipython_cmd, cmdargs, full_fname) #print >> sys.stderr, 'FULL CMD:', full_cmd # dbg - return genutils.getoutputerror(full_cmd) + return getoutputerror(full_cmd) def ipexec_validate(fname, expected_out, expected_err='', diff --git a/IPython/testing/util.py b/IPython/testing/util.py index 5c4253f..b8de077 100644 --- a/IPython/testing/util.py +++ b/IPython/testing/util.py @@ -1,23 +1,24 @@ # encoding: utf-8 """This file contains utility classes for performing tests with Deferreds. """ -__docformat__ = "restructuredtext en" -#------------------------------------------------------------------------------- -# Copyright (C) 2005 Fernando Perez -# Brian E Granger -# Benjamin Ragan-Kelley +#----------------------------------------------------------------------------- +# Copyright (C) 2009 The IPython Development Team # # Distributed under the terms of the BSD License. The full license is in # the file COPYING, distributed as part of this software. -#------------------------------------------------------------------------------- +#----------------------------------------------------------------------------- -#------------------------------------------------------------------------------- +#----------------------------------------------------------------------------- # Imports -#------------------------------------------------------------------------------- +#----------------------------------------------------------------------------- from twisted.trial import unittest from twisted.internet import defer +#----------------------------------------------------------------------------- +# Classes and functions +#----------------------------------------------------------------------------- + class DeferredTestCase(unittest.TestCase): def assertDeferredEquals(self, deferred, expectedResult, diff --git a/IPython/utils/DPyGetOpt.py b/IPython/utils/DPyGetOpt.py deleted file mode 100644 index f4b918e..0000000 --- a/IPython/utils/DPyGetOpt.py +++ /dev/null @@ -1,690 +0,0 @@ -# -*- coding: utf-8 -*- -"""DPyGetOpt -- Demiurge Python GetOptions Module - -This module is modeled after perl's Getopt::Long module-- which -is, in turn, modeled after GNU's extended getopt() function. - -Upon instantiation, the option specification should be a sequence -(list) of option definitions. - -Options that take no arguments should simply contain the name of -the option. If a ! is post-pended, the option can be negated by -prepending 'no'; ie 'debug!' specifies that -debug and -nodebug -should be accepted. - -Mandatory arguments to options are specified using a postpended -'=' + a type specifier. '=s' specifies a mandatory string -argument, '=i' specifies a mandatory integer argument, and '=f' -specifies a mandatory real number. In all cases, the '=' can be -substituted with ':' to specify that the argument is optional. - -Dashes '-' in option names are allowed. - -If an option has the character '@' postpended (after the -argumentation specification), it can appear multiple times within -each argument list that is processed. The results will be stored -in a list. - -The option name can actually be a list of names separated by '|' -characters; ie-- 'foo|bar|baz=f@' specifies that all -foo, -bar, -and -baz options that appear on within the parsed argument list -must have a real number argument and that the accumulated list -of values will be available under the name 'foo' -""" - -#***************************************************************************** -# -# Copyright (c) 2001 Bill Bumgarner -# -# -# Published under the terms of the MIT license, hereby reproduced: -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to -# deal in the Software without restriction, including without limitation the -# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or -# sell copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# -#***************************************************************************** - -__author__ = 'Bill Bumgarner ' -__license__ = 'MIT' -__version__ = '1.2' - -# Modified to use re instead of regex and regsub modules. -# 2001/5/7, Jonathan Hogg - -import re -import string -import sys -import types - -class Error(Exception): - """Base class for exceptions in the DPyGetOpt module.""" - -class ArgumentError(Error): - """Exception indicating an error in the arguments passed to - DPyGetOpt.processArguments.""" - -class SpecificationError(Error): - """Exception indicating an error with an option specification.""" - -class TerminationError(Error): - """Exception indicating an error with an option processing terminator.""" - -specificationExpr = re.compile('(?P.)(?P.)(?P@?)') - -ArgRequired = 'Requires an Argument' -ArgOptional = 'Argument Optional' - -# The types modules is not used for these identifiers because there -# is no identifier for 'boolean' or 'generic' -StringArgType = 'String Argument Type' -IntegerArgType = 'Integer Argument Type' -RealArgType = 'Real Argument Type' -BooleanArgType = 'Boolean Argument Type' -GenericArgType = 'Generic Argument Type' - -# dictionary of conversion functions-- boolean and generic options -# do not accept arguments and do not need conversion functions; -# the identity function is used purely for convenience. -ConversionFunctions = { - StringArgType : lambda x: x, - IntegerArgType : string.atoi, - RealArgType : string.atof, - BooleanArgType : lambda x: x, - GenericArgType : lambda x: x, - } - -class DPyGetOpt: - - def __init__(self, spec = None, terminators = ['--']): - """ - Declare and intialize instance variables - - Yes, declaration is not necessary... but one of the things - I sorely miss from C/Obj-C is the concept of having an - interface definition that clearly declares all instance - variables and methods without providing any implementation - details. it is a useful reference! - - all instance variables are initialized to 0/Null/None of - the appropriate type-- not even the default value... - """ - -# sys.stderr.write(string.join(spec) + "\n") - - self.allowAbbreviations = 1 # boolean, 1 if abbreviations will - # be expanded - self.freeValues = [] # list, contains free values - self.ignoreCase = 0 # boolean, YES if ignoring case - self.needsParse = 0 # boolean, YES if need to reparse parameter spec - self.optionNames = {} # dict, all option names-- value is index of tuple - self.optionStartExpr = None # regexp defining the start of an option (ie; '-', '--') - self.optionTuples = [] # list o' tuples containing defn of options AND aliases - self.optionValues = {} # dict, option names (after alias expansion) -> option value(s) - self.orderMixed = 0 # boolean, YES if options can be mixed with args - self.posixCompliance = 0 # boolean, YES indicates posix like behaviour - self.spec = [] # list, raw specs (in case it must be reparsed) - self.terminators = terminators # list, strings that terminate argument processing - self.termValues = [] # list, values after terminator - self.terminator = None # full name of terminator that ended - # option processing - - # set up defaults - self.setPosixCompliance() - self.setIgnoreCase() - self.setAllowAbbreviations() - - # parse spec-- if present - if spec: - self.parseConfiguration(spec) - - def setPosixCompliance(self, aFlag = 0): - """ - Enables and disables posix compliance. - - When enabled, '+' can be used as an option prefix and free - values can be mixed with options. - """ - self.posixCompliance = aFlag - self.needsParse = 1 - - if self.posixCompliance: - self.optionStartExpr = re.compile('(--|-)(?P