From 5016a14f2943e5044ed87b49964820ebf7c27135 2010-08-12 17:29:56 From: Brian Granger Date: 2010-08-12 17:29:56 Subject: [PATCH] Merge branch 'master' into kernelmanager Conflicts: IPython/utils/traitlets.py --- diff --git a/IPython/__init__.py b/IPython/__init__.py index ff3156b..aea962b 100755 --- a/IPython/__init__.py +++ b/IPython/__init__.py @@ -24,8 +24,8 @@ import sys # Setup everything #----------------------------------------------------------------------------- -if sys.version[0:3] < '2.5': - raise ImportError('Python Version 2.5 or above is required for IPython.') +if sys.version[0:3] < '2.6': + raise ImportError('Python Version 2.6 or above is required for IPython.') # Make it easy to import extensions - they are always directly on pythonpath. diff --git a/IPython/config/configurable.py b/IPython/config/configurable.py new file mode 100755 index 0000000..18cb050 --- /dev/null +++ b/IPython/config/configurable.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python +# encoding: utf-8 +""" +A base class for objects that are configurable. + +Authors: + +* Brian Granger +* Fernando Perez +""" + +#----------------------------------------------------------------------------- +# Copyright (C) 2008-2010 The IPython Development Team +# +# Distributed under the terms of the BSD License. The full license is in +# the file COPYING, distributed as part of this software. +#----------------------------------------------------------------------------- + +#----------------------------------------------------------------------------- +# Imports +#----------------------------------------------------------------------------- + +from copy import deepcopy +import datetime +from weakref import WeakValueDictionary + +from IPython.utils.importstring import import_item +from loader import Config +from IPython.utils.traitlets import HasTraits, Instance + + +#----------------------------------------------------------------------------- +# Helper classes for Configurables +#----------------------------------------------------------------------------- + + +class ConfigurableError(Exception): + pass + + +#----------------------------------------------------------------------------- +# Configurable implementation +#----------------------------------------------------------------------------- + + +class Configurable(HasTraits): + + config = Instance(Config,(),{}) + created = None + + def __init__(self, **kwargs): + """Create a conigurable given a config config. + + Parameters + ---------- + config : Config + If this is empty, default values are used. If config is a + :class:`Config` instance, it will be used to configure the + instance. + + Notes + ----- + Subclasses of Configurable must call the :meth:`__init__` method of + :class:`Configurable` *before* doing anything else and using + :func:`super`:: + + class MyConfigurable(Configurable): + def __init__(self, config=None): + super(MyConfigurable, self).__init__(config) + # Then any other code you need to finish initialization. + + This ensures that instances will be configured properly. + """ + config = kwargs.pop('config', None) + if config is not None: + # We used to deepcopy, but for now we are trying to just save + # by reference. This *could* have side effects as all components + # will share config. In fact, I did find such a side effect in + # _config_changed below. If a config attribute value was a mutable type + # all instances of a component were getting the same copy, effectively + # making that a class attribute. + # self.config = deepcopy(config) + self.config = config + # This should go second so individual keyword arguments override + # the values in config. + super(Configurable, self).__init__(**kwargs) + self.created = datetime.datetime.now() + + #------------------------------------------------------------------------- + # Static trait notifiations + #------------------------------------------------------------------------- + + def _config_changed(self, name, old, new): + """Update all the class traits having ``config=True`` as metadata. + + For any class trait with a ``config`` metadata attribute that is + ``True``, we update the trait with the value of the corresponding + config entry. + """ + # Get all traits with a config metadata entry that is True + traits = self.traits(config=True) + + # We auto-load config section for this class as well as any parent + # classes that are Configurable subclasses. This starts with Configurable + # and works down the mro loading the config for each section. + section_names = [cls.__name__ for cls in \ + reversed(self.__class__.__mro__) if + issubclass(cls, Configurable) and issubclass(self.__class__, cls)] + + for sname in section_names: + # Don't do a blind getattr as that would cause the config to + # dynamically create the section with name self.__class__.__name__. + if new._has_section(sname): + my_config = new[sname] + for k, v in traits.items(): + # Don't allow traitlets with config=True to start with + # uppercase. Otherwise, they are confused with Config + # subsections. But, developers shouldn't have uppercase + # attributes anyways! (PEP 6) + if k[0].upper()==k[0] and not k.startswith('_'): + raise ConfigurableError('Configurable traitlets with ' + 'config=True must start with a lowercase so they are ' + 'not confused with Config subsections: %s.%s' % \ + (self.__class__.__name__, k)) + try: + # Here we grab the value from the config + # If k has the naming convention of a config + # section, it will be auto created. + config_value = my_config[k] + except KeyError: + pass + else: + # print "Setting %s.%s from %s.%s=%r" % \ + # (self.__class__.__name__,k,sname,k,config_value) + # We have to do a deepcopy here if we don't deepcopy the entire + # config object. If we don't, a mutable config_value will be + # shared by all instances, effectively making it a class attribute. + setattr(self, k, deepcopy(config_value)) + diff --git a/IPython/config/tests/test_configurable.py b/IPython/config/tests/test_configurable.py new file mode 100644 index 0000000..d57c36d --- /dev/null +++ b/IPython/config/tests/test_configurable.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python +# encoding: utf-8 +""" +Tests for IPython.config.configurable + +Authors: + +* Brian Granger +* Fernando Perez (design help) +""" + +#----------------------------------------------------------------------------- +# Copyright (C) 2008-2010 The IPython Development Team +# +# Distributed under the terms of the BSD License. The full license is in +# the file COPYING, distributed as part of this software. +#----------------------------------------------------------------------------- + +#----------------------------------------------------------------------------- +# Imports +#----------------------------------------------------------------------------- + +from unittest import TestCase + +from IPython.config.configurable import Configurable, ConfigurableError +from IPython.utils.traitlets import ( + TraitError, Int, Float, Str +) +from IPython.config.loader import Config + + +#----------------------------------------------------------------------------- +# Test cases +#----------------------------------------------------------------------------- + + +class MyConfigurable(Configurable): + a = Int(1, config=True) + b = Float(1.0, config=True) + c = Str('no config') + + +class Foo(Configurable): + a = Int(0, config=True) + b = Str('nope', config=True) + + +class Bar(Foo): + b = Str('gotit', config=False) + c = Float(config=True) + + +class TestConfigurableConfig(TestCase): + + def test_default(self): + c1 = Configurable() + c2 = Configurable(config=c1.config) + c3 = Configurable(config=c2.config) + self.assertEquals(c1.config, c2.config) + self.assertEquals(c2.config, c3.config) + + def test_custom(self): + config = Config() + config.foo = 'foo' + config.bar = 'bar' + c1 = Configurable(config=config) + c2 = Configurable(config=c1.config) + c3 = Configurable(config=c2.config) + self.assertEquals(c1.config, config) + self.assertEquals(c2.config, config) + self.assertEquals(c3.config, config) + # Test that copies are not made + self.assert_(c1.config is config) + self.assert_(c2.config is config) + self.assert_(c3.config is config) + self.assert_(c1.config is c2.config) + self.assert_(c2.config is c3.config) + + def test_inheritance(self): + config = Config() + config.MyConfigurable.a = 2 + config.MyConfigurable.b = 2.0 + c1 = MyConfigurable(config=config) + c2 = MyConfigurable(config=c1.config) + self.assertEquals(c1.a, config.MyConfigurable.a) + self.assertEquals(c1.b, config.MyConfigurable.b) + self.assertEquals(c2.a, config.MyConfigurable.a) + self.assertEquals(c2.b, config.MyConfigurable.b) + + def test_parent(self): + config = Config() + config.Foo.a = 10 + config.Foo.b = "wow" + config.Bar.b = 'later' + config.Bar.c = 100.0 + f = Foo(config=config) + b = Bar(config=f.config) + self.assertEquals(f.a, 10) + self.assertEquals(f.b, 'wow') + self.assertEquals(b.b, 'gotit') + self.assertEquals(b.c, 100.0) + + def test_override1(self): + config = Config() + config.MyConfigurable.a = 2 + config.MyConfigurable.b = 2.0 + c = MyConfigurable(a=3, config=config) + self.assertEquals(c.a, 3) + self.assertEquals(c.b, config.MyConfigurable.b) + self.assertEquals(c.c, 'no config') + + def test_override2(self): + config = Config() + config.Foo.a = 1 + config.Bar.b = 'or' # Up above b is config=False, so this won't do it. + config.Bar.c = 10.0 + c = Bar(config=config) + self.assertEquals(c.a, config.Foo.a) + self.assertEquals(c.b, 'gotit') + self.assertEquals(c.c, config.Bar.c) + c = Bar(a=2, b='and', c=20.0, config=config) + self.assertEquals(c.a, 2) + self.assertEquals(c.b, 'and') + self.assertEquals(c.c, 20.0) diff --git a/IPython/config/tests/test_imports.py b/IPython/config/tests/test_imports.py deleted file mode 100644 index 84fb531..0000000 --- a/IPython/config/tests/test_imports.py +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env python -# encoding: utf-8 - - - diff --git a/IPython/core/alias.py b/IPython/core/alias.py index 1afe854..506097a 100644 --- a/IPython/core/alias.py +++ b/IPython/core/alias.py @@ -1,10 +1,11 @@ #!/usr/bin/env python # encoding: utf-8 """ -IPython's alias component +System command aliases. Authors: +* Fernando Perez * Brian Granger """ @@ -25,10 +26,10 @@ import os import re import sys -from IPython.core.component import Component +from IPython.config.configurable import Configurable from IPython.core.splitinput import split_user_input -from IPython.utils.traitlets import List +from IPython.utils.traitlets import List, Instance from IPython.utils.autoattr import auto_attr from IPython.utils.warn import warn, error @@ -99,23 +100,18 @@ class InvalidAliasError(AliasError): #----------------------------------------------------------------------------- -class AliasManager(Component): +class AliasManager(Configurable): default_aliases = List(default_aliases(), config=True) user_aliases = List(default_value=[], config=True) + shell = Instance('IPython.core.iplib.InteractiveShellABC') - def __init__(self, parent, config=None): - super(AliasManager, self).__init__(parent, config=config) + def __init__(self, shell=None, config=None): + super(AliasManager, self).__init__(shell=shell, config=config) self.alias_table = {} self.exclude_aliases() self.init_aliases() - @auto_attr - def shell(self): - return Component.get_instances( - root=self.root, - klass='IPython.core.iplib.InteractiveShell')[0] - def __contains__(self, name): if name in self.alias_table: return True diff --git a/IPython/core/application.py b/IPython/core/application.py index 7fdea83..b6c25c6 100644 --- a/IPython/core/application.py +++ b/IPython/core/application.py @@ -6,7 +6,7 @@ All top-level applications should use the classes in this module for handling configuration and creating componenets. The job of an :class:`Application` is to create the master configuration -object and then create the components, passing the config to them. +object and then create the configurable objects, passing the config to them. Authors: @@ -76,7 +76,7 @@ class BaseAppConfigLoader(ArgParseConfigLoader): class Application(object): - """Load a config, construct components and set them running. + """Load a config, construct configurables and set them running. The configuration of an application can be done via three different Config objects, which are loaded and ultimately merged into a single one used @@ -113,7 +113,7 @@ class Application(object): file_config = None #: Read from the system's command line flags. command_line_config = None - #: The final config that will be passed to the component. + #: The final config that will be passed to the main object. master_config = None #: A reference to the argv to be used (typically ends up being sys.argv[1:]) argv = None @@ -223,10 +223,10 @@ class Application(object): """Create defaults that can't be set elsewhere. For the most part, we try to set default in the class attributes - of Components. But, defaults the top-level Application (which is - not a HasTraits or Component) are not set in this way. Instead + of Configurables. But, defaults the top-level Application (which is + not a HasTraits or Configurables) are not set in this way. Instead we set them here. The Global section is for variables like this that - don't belong to a particular component. + don't belong to a particular configurable. """ c = Config() c.Global.ipython_dir = get_ipython_dir() @@ -418,8 +418,8 @@ class Application(object): pass def construct(self): - """Construct the main components that make up this app.""" - self.log.debug("Constructing components for application") + """Construct the main objects that make up this app.""" + self.log.debug("Constructing main objects for application") def post_construct(self): """Do actions after construct, but before starting the app.""" diff --git a/IPython/core/builtin_trap.py b/IPython/core/builtin_trap.py index 8021c5a..758d514 100755 --- a/IPython/core/builtin_trap.py +++ b/IPython/core/builtin_trap.py @@ -21,10 +21,10 @@ Authors: import __builtin__ -from IPython.core.component import Component +from IPython.config.configurable import Configurable from IPython.core.quitter import Quitter -from IPython.utils.autoattr import auto_attr +from IPython.utils.traitlets import Instance #----------------------------------------------------------------------------- # Classes and functions @@ -35,20 +35,17 @@ class __BuiltinUndefined(object): pass BuiltinUndefined = __BuiltinUndefined() -class BuiltinTrap(Component): +class BuiltinTrap(Configurable): - def __init__(self, parent): - super(BuiltinTrap, self).__init__(parent, None, None) + shell = Instance('IPython.core.iplib.InteractiveShellABC') + + def __init__(self, shell=None): + super(BuiltinTrap, self).__init__(shell=shell, config=None) self._orig_builtins = {} # We define this to track if a single BuiltinTrap is nested. # Only turn off the trap when the outermost call to __exit__ is made. self._nested_level = 0 - - @auto_attr - def shell(self): - return Component.get_instances( - root=self.root, - klass='IPython.core.iplib.InteractiveShell')[0] + self.shell = shell def __enter__(self): if self._nested_level == 0: diff --git a/IPython/core/component.py b/IPython/core/component.py deleted file mode 100755 index 98b6508..0000000 --- a/IPython/core/component.py +++ /dev/null @@ -1,346 +0,0 @@ -#!/usr/bin/env python -# encoding: utf-8 -""" -A lightweight component system for IPython. - -Authors: - -* Brian Granger -* Fernando Perez -""" - -#----------------------------------------------------------------------------- -# Copyright (C) 2008-2009 The IPython Development Team -# -# Distributed under the terms of the BSD License. The full license is in -# the file COPYING, distributed as part of this software. -#----------------------------------------------------------------------------- - -#----------------------------------------------------------------------------- -# Imports -#----------------------------------------------------------------------------- - -from copy import deepcopy -import datetime -from weakref import WeakValueDictionary - -from IPython.utils.importstring import import_item -from IPython.config.loader import Config -from IPython.utils.traitlets import ( - HasTraits, MetaHasTraits, Instance, This -) - - -#----------------------------------------------------------------------------- -# Helper classes for Components -#----------------------------------------------------------------------------- - - -class ComponentError(Exception): - pass - -class MetaComponentTracker(type): - """A metaclass that tracks instances of Components and its subclasses.""" - - def __init__(cls, name, bases, d): - super(MetaComponentTracker, cls).__init__(name, bases, d) - cls.__instance_refs = WeakValueDictionary() - cls.__numcreated = 0 - - def __call__(cls, *args, **kw): - """Called when a class is called (instantiated)!!! - - When a Component or subclass is instantiated, this is called and - the instance is saved in a WeakValueDictionary for tracking. - """ - instance = cls.__new__(cls, *args, **kw) - - # Register the instance before __init__ is called so get_instances - # works inside __init__ methods! - indices = cls.register_instance(instance) - - # This is in a try/except because of the __init__ method fails, the - # instance is discarded and shouldn't be tracked. - try: - if isinstance(instance, cls): - cls.__init__(instance, *args, **kw) - except: - # Unregister the instance because __init__ failed! - cls.unregister_instances(indices) - raise - else: - return instance - - def register_instance(cls, instance): - """Register instance with cls and its subclasses.""" - # indices is a list of the keys used to register the instance - # with. This list is needed if the instance needs to be unregistered. - indices = [] - for c in cls.__mro__: - if issubclass(cls, c) and issubclass(c, Component): - c.__numcreated += 1 - indices.append(c.__numcreated) - c.__instance_refs[c.__numcreated] = instance - else: - break - return indices - - def unregister_instances(cls, indices): - """Unregister instance with cls and its subclasses.""" - for c, index in zip(cls.__mro__, indices): - try: - del c.__instance_refs[index] - except KeyError: - pass - - def clear_instances(cls): - """Clear all instances tracked by cls.""" - cls.__instance_refs.clear() - cls.__numcreated = 0 - - def get_instances(cls, name=None, root=None, klass=None): - """Get all instances of cls and its subclasses. - - Parameters - ---------- - name : str - Limit to components with this name. - root : Component or subclass - Limit to components having this root. - klass : class or str - Limits to instances of the class or its subclasses. If a str - is given ut must be in the form 'foo.bar.MyClass'. The str - form of this argument is useful for forward declarations. - """ - if klass is not None: - if isinstance(klass, basestring): - klass = import_item(klass) - # Limit search to instances of klass for performance - if issubclass(klass, Component): - return klass.get_instances(name=name, root=root) - instances = cls.__instance_refs.values() - if name is not None: - instances = [i for i in instances if i.name == name] - if klass is not None: - instances = [i for i in instances if isinstance(i, klass)] - if root is not None: - instances = [i for i in instances if i.root == root] - return instances - - def get_instances_by_condition(cls, call, name=None, root=None, - klass=None): - """Get all instances of cls, i such that call(i)==True. - - This also takes the ``name`` and ``root`` and ``classname`` - arguments of :meth:`get_instance` - """ - return [i for i in cls.get_instances(name, root, klass) if call(i)] - - -def masquerade_as(instance, cls): - """Let instance masquerade as an instance of cls. - - Sometimes, such as in testing code, it is useful to let a class - masquerade as another. Python, being duck typed, allows this by - default. But, instances of components are tracked by their class type. - - After calling this, ``cls.get_instances()`` will return ``instance``. This - does not, however, cause ``isinstance(instance, cls)`` to return ``True``. - - Parameters - ---------- - instance : an instance of a Component or Component subclass - The instance that will pretend to be a cls. - cls : subclass of Component - The Component subclass that instance will pretend to be. - """ - cls.register_instance(instance) - - -class __ComponentNameGenerator(object): - """A Singleton to generate unique component names.""" - - def __init__(self, prefix): - self.prefix = prefix - self.i = 0 - - def __call__(self): - count = self.i - self.i += 1 - return "%s%s" % (self.prefix, count) - - -ComponentNameGenerator = __ComponentNameGenerator('ipython.component') - - -class MetaComponent(MetaHasTraits, MetaComponentTracker): - pass - - -#----------------------------------------------------------------------------- -# Component implementation -#----------------------------------------------------------------------------- - - -class Component(HasTraits): - - __metaclass__ = MetaComponent - - # Traits are fun! - config = Instance(Config,(),{}) - parent = This() - root = This() - created = None - - def __init__(self, parent, name=None, config=None): - """Create a component given a parent and possibly and name and config. - - Parameters - ---------- - parent : Component subclass - The parent in the component graph. The parent is used - to get the root of the component graph. - name : str - The unique name of the component. If empty, then a unique - one will be autogenerated. - config : Config - If this is empty, self.config = parent.config, otherwise - self.config = config and root.config is ignored. This argument - should only be used to *override* the automatic inheritance of - parent.config. If a caller wants to modify parent.config - (not override), the caller should make a copy and change - attributes and then pass the copy to this argument. - - Notes - ----- - Subclasses of Component must call the :meth:`__init__` method of - :class:`Component` *before* doing anything else and using - :func:`super`:: - - class MyComponent(Component): - def __init__(self, parent, name=None, config=None): - super(MyComponent, self).__init__(parent, name, config) - # Then any other code you need to finish initialization. - - This ensures that the :attr:`parent`, :attr:`name` and :attr:`config` - attributes are handled properly. - """ - super(Component, self).__init__() - self._children = [] - if name is None: - self.name = ComponentNameGenerator() - else: - self.name = name - self.root = self # This is the default, it is set when parent is set - self.parent = parent - if config is not None: - self.config = config - # We used to deepcopy, but for now we are trying to just save - # by reference. This *could* have side effects as all components - # will share config. In fact, I did find such a side effect in - # _config_changed below. If a config attribute value was a mutable type - # all instances of a component were getting the same copy, effectively - # making that a class attribute. - # self.config = deepcopy(config) - else: - if self.parent is not None: - self.config = self.parent.config - # We used to deepcopy, but for now we are trying to just save - # by reference. This *could* have side effects as all components - # will share config. In fact, I did find such a side effect in - # _config_changed below. If a config attribute value was a mutable type - # all instances of a component were getting the same copy, effectively - # making that a class attribute. - # self.config = deepcopy(self.parent.config) - - self.created = datetime.datetime.now() - - #------------------------------------------------------------------------- - # Static trait notifiations - #------------------------------------------------------------------------- - - def _parent_changed(self, name, old, new): - if old is not None: - old._remove_child(self) - if new is not None: - new._add_child(self) - - if new is None: - self.root = self - else: - self.root = new.root - - def _root_changed(self, name, old, new): - if self.parent is None: - if not (new is self): - raise ComponentError("Root not self, but parent is None.") - else: - if not self.parent.root is new: - raise ComponentError("Error in setting the root attribute: " - "root != parent.root") - - def _config_changed(self, name, old, new): - """Update all the class traits having ``config=True`` as metadata. - - For any class trait with a ``config`` metadata attribute that is - ``True``, we update the trait with the value of the corresponding - config entry. - """ - # Get all traits with a config metadata entry that is True - traits = self.traits(config=True) - - # We auto-load config section for this class as well as any parent - # classes that are Component subclasses. This starts with Component - # and works down the mro loading the config for each section. - section_names = [cls.__name__ for cls in \ - reversed(self.__class__.__mro__) if - issubclass(cls, Component) and issubclass(self.__class__, cls)] - - for sname in section_names: - # Don't do a blind getattr as that would cause the config to - # dynamically create the section with name self.__class__.__name__. - if new._has_section(sname): - my_config = new[sname] - for k, v in traits.items(): - # Don't allow traitlets with config=True to start with - # uppercase. Otherwise, they are confused with Config - # subsections. But, developers shouldn't have uppercase - # attributes anyways! (PEP 6) - if k[0].upper()==k[0] and not k.startswith('_'): - raise ComponentError('Component traitlets with ' - 'config=True must start with a lowercase so they are ' - 'not confused with Config subsections: %s.%s' % \ - (self.__class__.__name__, k)) - try: - # Here we grab the value from the config - # If k has the naming convention of a config - # section, it will be auto created. - config_value = my_config[k] - except KeyError: - pass - else: - # print "Setting %s.%s from %s.%s=%r" % \ - # (self.__class__.__name__,k,sname,k,config_value) - # We have to do a deepcopy here if we don't deepcopy the entire - # config object. If we don't, a mutable config_value will be - # shared by all instances, effectively making it a class attribute. - setattr(self, k, deepcopy(config_value)) - - @property - def children(self): - """A list of all my child components.""" - return self._children - - def _remove_child(self, child): - """A private method for removing children components.""" - if child in self._children: - index = self._children.index(child) - del self._children[index] - - def _add_child(self, child): - """A private method for adding children components.""" - if child not in self._children: - self._children.append(child) - - def __repr__(self): - return "<%s('%s')>" % (self.__class__.__name__, self.name) diff --git a/IPython/core/display_trap.py b/IPython/core/display_trap.py index d5e5e83..d835b54 100644 --- a/IPython/core/display_trap.py +++ b/IPython/core/display_trap.py @@ -22,34 +22,30 @@ Authors: import sys -from IPython.core.component import Component +from IPython.config.configurable import Configurable +from IPython.utils.traitlets import Any #----------------------------------------------------------------------------- # Classes and functions #----------------------------------------------------------------------------- -class DisplayTrap(Component): +class DisplayTrap(Configurable): """Object to manage sys.displayhook. This came from IPython.core.kernel.display_hook, but is simplified (no callbacks or formatters) until more of the core is refactored. """ - def __init__(self, parent, hook): - super(DisplayTrap, self).__init__(parent, None, None) - self.hook = hook + hook = Any + + def __init__(self, hook=None): + super(DisplayTrap, self).__init__(hook=hook, config=None) self.old_hook = None # We define this to track if a single BuiltinTrap is nested. # Only turn off the trap when the outermost call to __exit__ is made. self._nested_level = 0 - # @auto_attr - # def shell(self): - # return Component.get_instances( - # root=self.root, - # klass='IPython.core.iplib.InteractiveShell')[0] - def __enter__(self): if self._nested_level == 0: self.set() diff --git a/IPython/core/extensions.py b/IPython/core/extensions.py new file mode 100644 index 0000000..bf1757e --- /dev/null +++ b/IPython/core/extensions.py @@ -0,0 +1,125 @@ +# encoding: utf-8 +"""A class for managing IPython extensions. + +Authors: + +* Brian Granger +""" + +#----------------------------------------------------------------------------- +# Copyright (C) 2010 The IPython Development Team +# +# Distributed under the terms of the BSD License. The full license is in +# the file COPYING, distributed as part of this software. +#----------------------------------------------------------------------------- + +#----------------------------------------------------------------------------- +# Imports +#----------------------------------------------------------------------------- + +import os +import sys + +from IPython.config.configurable import Configurable +from IPython.utils.traitlets import Instance + +#----------------------------------------------------------------------------- +# Main class +#----------------------------------------------------------------------------- + +class ExtensionManager(Configurable): + """A class to manage IPython extensions. + + An IPython extension is an importable Python module that has + a function with the signature:: + + def load_ipython_extension(ipython): + # Do things with ipython + + This function is called after your extension is imported and the + currently active :class:`InteractiveShell` instance is passed as + the only argument. You can do anything you want with IPython at + that point, including defining new magic and aliases, adding new + components, etc. + + The :func:`load_ipython_extension` will be called again is you + load or reload the extension again. It is up to the extension + author to add code to manage that. + + You can put your extension modules anywhere you want, as long as + they can be imported by Python's standard import mechanism. However, + to make it easy to write extensions, you can also put your extensions + in ``os.path.join(self.ipython_dir, 'extensions')``. This directory + is added to ``sys.path`` automatically. + """ + + shell = Instance('IPython.core.iplib.InteractiveShellABC') + + def __init__(self, shell=None, config=None): + super(ExtensionManager, self).__init__(shell=shell, config=config) + self.shell.on_trait_change( + self._on_ipython_dir_changed, 'ipython_dir' + ) + + def __del__(self): + self.shell.on_trait_change( + self._on_ipython_dir_changed, 'ipython_dir', remove=True + ) + + @property + def ipython_extension_dir(self): + return os.path.join(self.shell.ipython_dir, u'extensions') + + def _on_ipython_dir_changed(self): + if not os.path.isdir(self.ipython_extension_dir): + os.makedirs(self.ipython_extension_dir, mode = 0777) + + def load_extension(self, module_str): + """Load an IPython extension by its module name. + + If :func:`load_ipython_extension` returns anything, this function + will return that object. + """ + from IPython.utils.syspathcontext import prepended_to_syspath + + if module_str not in sys.modules: + with prepended_to_syspath(self.ipython_extension_dir): + __import__(module_str) + mod = sys.modules[module_str] + return self._call_load_ipython_extension(mod) + + def unload_extension(self, module_str): + """Unload an IPython extension by its module name. + + This function looks up the extension's name in ``sys.modules`` and + simply calls ``mod.unload_ipython_extension(self)``. + """ + if module_str in sys.modules: + mod = sys.modules[module_str] + self._call_unload_ipython_extension(mod) + + def reload_extension(self, module_str): + """Reload an IPython extension by calling reload. + + If the module has not been loaded before, + :meth:`InteractiveShell.load_extension` is called. Otherwise + :func:`reload` is called and then the :func:`load_ipython_extension` + function of the module, if it exists is called. + """ + from IPython.utils.syspathcontext import prepended_to_syspath + + with prepended_to_syspath(self.ipython_extension_dir): + if module_str in sys.modules: + mod = sys.modules[module_str] + reload(mod) + self._call_load_ipython_extension(mod) + else: + self.load_extension(module_str) + + def _call_load_ipython_extension(self, mod): + if hasattr(mod, 'load_ipython_extension'): + return mod.load_ipython_extension(self.shell) + + def _call_unload_ipython_extension(self, mod): + if hasattr(mod, 'unload_ipython_extension'): + return mod.unload_ipython_extension(self.shell) diff --git a/IPython/core/ipapi.py b/IPython/core/ipapi.py index 461aa5a..24fe36d 100644 --- a/IPython/core/ipapi.py +++ b/IPython/core/ipapi.py @@ -24,13 +24,7 @@ has been made into a component, this module will be sent to deathrow. def get(): - """Get the most recently created InteractiveShell instance.""" + """Get the global InteractiveShell instance.""" from IPython.core.iplib import InteractiveShell - insts = InteractiveShell.get_instances() - if len(insts)==0: - return None - most_recent = insts[0] - for inst in insts[1:]: - if inst.created > most_recent.created: - most_recent = inst - return most_recent + return InteractiveShell.instance() + diff --git a/IPython/core/ipapp.py b/IPython/core/ipapp.py index 1dcbd3d..6733afe 100755 --- a/IPython/core/ipapp.py +++ b/IPython/core/ipapp.py @@ -475,8 +475,8 @@ class IPythonApp(Application): # But that might be the place for them sys.path.insert(0, '') - # Create an InteractiveShell instance - self.shell = InteractiveShell(None, self.master_config) + # Create an InteractiveShell instance. + self.shell = InteractiveShell.instance(config=self.master_config) def post_construct(self): """Do actions after construct, but before starting the app.""" @@ -543,7 +543,7 @@ class IPythonApp(Application): def _load_extensions(self): """Load all IPython extensions in Global.extensions. - This uses the :meth:`InteractiveShell.load_extensions` to load all + This uses the :meth:`ExtensionManager.load_extensions` to load all the extensions listed in ``self.master_config.Global.extensions``. """ try: @@ -553,7 +553,7 @@ class IPythonApp(Application): for ext in extensions: try: self.log.info("Loading IPython extension: %s" % ext) - self.shell.load_extension(ext) + self.shell.extension_manager.load_extension(ext) except: self.log.warn("Error in loading extension: %s" % ext) self.shell.showtraceback() diff --git a/IPython/core/iplib.py b/IPython/core/iplib.py index ae56cfb..a098db6 100644 --- a/IPython/core/iplib.py +++ b/IPython/core/iplib.py @@ -1,12 +1,10 @@ # -*- coding: utf-8 -*- -""" -Main IPython Component -""" +"""Main IPython class.""" #----------------------------------------------------------------------------- # Copyright (C) 2001 Janko Hauser # Copyright (C) 2001-2007 Fernando Perez. -# Copyright (C) 2008-2009 The IPython Development Team +# Copyright (C) 2008-2010 The IPython Development Team # # Distributed under the terms of the BSD License. The full license is in # the file COPYING, distributed as part of this software. @@ -20,6 +18,7 @@ from __future__ import with_statement from __future__ import absolute_import import __builtin__ +import abc import bdb import codeop import exceptions @@ -38,12 +37,14 @@ from IPython.core import shadowns from IPython.core import ultratb from IPython.core.alias import AliasManager from IPython.core.builtin_trap import BuiltinTrap -from IPython.core.component import Component +from IPython.config.configurable import Configurable from IPython.core.display_trap import DisplayTrap from IPython.core.error import TryNext, UsageError +from IPython.core.extensions import ExtensionManager from IPython.core.fakemodule import FakeModule, init_fakemod_dict from IPython.core.logger import Logger from IPython.core.magic import Magic +from IPython.core.plugin import PluginManager from IPython.core.prefilter import PrefilterManager from IPython.core.prompts import CachedOutput from IPython.core.usage import interactive_usage, default_banner @@ -69,7 +70,7 @@ from IPython.utils.syspathcontext import prepended_to_syspath from IPython.utils.terminal import toggle_set_term_title, set_term_title from IPython.utils.warn import warn, error, fatal from IPython.utils.traitlets import ( - Int, Str, CBool, CaselessStrEnum, Enum, List, Unicode + Int, Str, CBool, CaselessStrEnum, Enum, List, Unicode, Instance ) # from IPython.utils import growl @@ -196,7 +197,7 @@ class SeparateStr(Str): #----------------------------------------------------------------------------- -class InteractiveShell(Component, Magic): +class InteractiveShell(Configurable, Magic): """An enhanced, interactive shell for Python.""" autocall = Enum((0,1,2), default_value=1, config=True) @@ -281,14 +282,22 @@ class InteractiveShell(Component, Magic): # Subclasses with thread support should override this as needed. isthreaded = False - def __init__(self, parent=None, config=None, ipython_dir=None, usage=None, + # Subcomponents of InteractiveShell + alias_manager = Instance('IPython.core.alias.AliasManager') + prefilter_manager = Instance('IPython.core.prefilter.PrefilterManager') + builtin_trap = Instance('IPython.core.builtin_trap.BuiltinTrap') + display_trap = Instance('IPython.core.display_trap.DisplayTrap') + extension_manager = Instance('IPython.core.extensions.ExtensionManager') + plugin_manager = Instance('IPython.core.plugin.PluginManager') + + def __init__(self, config=None, ipython_dir=None, usage=None, user_ns=None, user_global_ns=None, banner1=None, banner2=None, display_banner=None, custom_exceptions=((),None)): # This is where traits with a config_key argument are updated # from the values on config. - super(InteractiveShell, self).__init__(parent, config=config) + super(InteractiveShell, self).__init__(config=config) # These are relatively independent and stateless self.init_ipython_dir(ipython_dir) @@ -334,8 +343,21 @@ class InteractiveShell(Component, Magic): self.init_reload_doctest() self.init_magics() self.init_pdb() + self.init_extension_manager() + self.init_plugin_manager() self.hooks.late_startup_hook() + @classmethod + def instance(cls, *args, **kwargs): + """Returns a global InteractiveShell instance.""" + if not hasattr(cls, "_instance"): + cls._instance = cls(*args, **kwargs) + return cls._instance + + @classmethod + def initialized(cls): + return hasattr(cls, "_instance") + def get_ipython(self): """Return the currently running IPython instance.""" return self @@ -353,12 +375,6 @@ class InteractiveShell(Component, Magic): def _ipython_dir_changed(self, name, new): if not os.path.isdir(new): os.makedirs(new, mode = 0777) - if not os.path.isdir(self.ipython_extension_dir): - os.makedirs(self.ipython_extension_dir, mode = 0777) - - @property - def ipython_extension_dir(self): - return os.path.join(self.ipython_dir, 'extensions') @property def usable_screen_length(self): @@ -494,7 +510,7 @@ class InteractiveShell(Component, Magic): self.magic_logstart() def init_builtins(self): - self.builtin_trap = BuiltinTrap(self) + self.builtin_trap = BuiltinTrap(shell=self) def init_inspector(self): # Object inspector @@ -523,7 +539,7 @@ class InteractiveShell(Component, Magic): pass def init_displayhook(self): - self.display_trap = DisplayTrap(self, self.outputcache) + self.display_trap = DisplayTrap(hook=self.outputcache) def init_reload_doctest(self): # Do a proper resetting of doctest, including the necessary displayhook @@ -1743,10 +1759,20 @@ class InteractiveShell(Component, Magic): #------------------------------------------------------------------------- def init_alias(self): - self.alias_manager = AliasManager(self, config=self.config) + self.alias_manager = AliasManager(shell=self, config=self.config) self.ns_table['alias'] = self.alias_manager.alias_table, #------------------------------------------------------------------------- + # Things related to extensions and plugins + #------------------------------------------------------------------------- + + def init_extension_manager(self): + self.extension_manager = ExtensionManager(shell=self, config=self.config) + + def init_plugin_manager(self): + self.plugin_manager = PluginManager(config=self.config) + + #------------------------------------------------------------------------- # Things related to the running of code #------------------------------------------------------------------------- @@ -2340,101 +2366,11 @@ class InteractiveShell(Component, Magic): return lineout #------------------------------------------------------------------------- - # Working with components - #------------------------------------------------------------------------- - - def get_component(self, name=None, klass=None): - """Fetch a component by name and klass in my tree.""" - c = Component.get_instances(root=self, name=name, klass=klass) - if len(c) == 0: - return None - if len(c) == 1: - return c[0] - else: - return c - - #------------------------------------------------------------------------- - # IPython extensions - #------------------------------------------------------------------------- - - def load_extension(self, module_str): - """Load an IPython extension by its module name. - - An IPython extension is an importable Python module that has - a function with the signature:: - - def load_ipython_extension(ipython): - # Do things with ipython - - This function is called after your extension is imported and the - currently active :class:`InteractiveShell` instance is passed as - the only argument. You can do anything you want with IPython at - that point, including defining new magic and aliases, adding new - components, etc. - - The :func:`load_ipython_extension` will be called again is you - load or reload the extension again. It is up to the extension - author to add code to manage that. - - You can put your extension modules anywhere you want, as long as - they can be imported by Python's standard import mechanism. However, - to make it easy to write extensions, you can also put your extensions - in ``os.path.join(self.ipython_dir, 'extensions')``. This directory - is added to ``sys.path`` automatically. - - If :func:`load_ipython_extension` returns anything, this function - will return that object. - """ - from IPython.utils.syspathcontext import prepended_to_syspath - - if module_str not in sys.modules: - with prepended_to_syspath(self.ipython_extension_dir): - __import__(module_str) - mod = sys.modules[module_str] - return self._call_load_ipython_extension(mod) - - def unload_extension(self, module_str): - """Unload an IPython extension by its module name. - - This function looks up the extension's name in ``sys.modules`` and - simply calls ``mod.unload_ipython_extension(self)``. - """ - if module_str in sys.modules: - mod = sys.modules[module_str] - self._call_unload_ipython_extension(mod) - - def reload_extension(self, module_str): - """Reload an IPython extension by calling reload. - - If the module has not been loaded before, - :meth:`InteractiveShell.load_extension` is called. Otherwise - :func:`reload` is called and then the :func:`load_ipython_extension` - function of the module, if it exists is called. - """ - from IPython.utils.syspathcontext import prepended_to_syspath - - with prepended_to_syspath(self.ipython_extension_dir): - if module_str in sys.modules: - mod = sys.modules[module_str] - reload(mod) - self._call_load_ipython_extension(mod) - else: - self.load_extension(module_str) - - def _call_load_ipython_extension(self, mod): - if hasattr(mod, 'load_ipython_extension'): - return mod.load_ipython_extension(self) - - def _call_unload_ipython_extension(self, mod): - if hasattr(mod, 'unload_ipython_extension'): - return mod.unload_ipython_extension(self) - - #------------------------------------------------------------------------- # Things related to the prefilter #------------------------------------------------------------------------- def init_prefilter(self): - self.prefilter_manager = PrefilterManager(self, config=self.config) + self.prefilter_manager = PrefilterManager(shell=self, config=self.config) # Ultimately this will be refactored in the new interpreter code, but # for now, we should expose the main prefilter method (there's legacy # code out there that may rely on this). @@ -2580,3 +2516,8 @@ class InteractiveShell(Component, Magic): self.restore_sys_module_state() +class InteractiveShellABC(object): + """An abstract base class for InteractiveShell.""" + __metaclass__ = abc.ABCMeta + +InteractiveShellABC.register(InteractiveShell) diff --git a/IPython/core/magic.py b/IPython/core/magic.py index 621743b..b44a41a 100644 --- a/IPython/core/magic.py +++ b/IPython/core/magic.py @@ -97,11 +97,11 @@ def compress_dhist(dh): # XXX - for some odd reason, if Magic is made a new-style class, we get errors # on construction of the main InteractiveShell object. Something odd is going -# on with super() calls, Component and the MRO... For now leave it as-is, but +# on with super() calls, Configurable and the MRO... For now leave it as-is, but # eventually this needs to be clarified. # BG: This is because InteractiveShell inherits from this, but is itself a -# Component. This messes up the MRO in some way. The fix is that we need to -# make Magic a component that InteractiveShell does not subclass. +# Configurable. This messes up the MRO in some way. The fix is that we need to +# make Magic a configurable that InteractiveShell does not subclass. class Magic: """Magic functions for InteractiveShell. @@ -3586,15 +3586,15 @@ Defaulting color scheme to 'NoColor'""" def magic_load_ext(self, module_str): """Load an IPython extension by its module name.""" - return self.load_extension(module_str) + return self.extension_manager.load_extension(module_str) def magic_unload_ext(self, module_str): """Unload an IPython extension by its module name.""" - self.unload_extension(module_str) + self.extension_manager.unload_extension(module_str) def magic_reload_ext(self, module_str): """Reload an IPython extension by its module name.""" - self.reload_extension(module_str) + self.extension_manager.reload_extension(module_str) @testdec.skip_doctest def magic_install_profiles(self, s): diff --git a/IPython/core/plugin.py b/IPython/core/plugin.py new file mode 100644 index 0000000..ee12abe --- /dev/null +++ b/IPython/core/plugin.py @@ -0,0 +1,51 @@ +# encoding: utf-8 +"""IPython plugins. + +Authors: + +* Brian Granger +""" + +#----------------------------------------------------------------------------- +# Copyright (C) 2010 The IPython Development Team +# +# Distributed under the terms of the BSD License. The full license is in +# the file COPYING, distributed as part of this software. +#----------------------------------------------------------------------------- + +#----------------------------------------------------------------------------- +# Imports +#----------------------------------------------------------------------------- + +from IPython.config.configurable import Configurable +from IPython.utils.traitlets import Dict + +#----------------------------------------------------------------------------- +# Main class +#----------------------------------------------------------------------------- + +class PluginManager(Configurable): + """A manager for IPython plugins.""" + + plugins = Dict({}) + + def __init__(self, config=None): + super(PluginManager, self).__init__(config=config) + + def register_plugin(self, name, plugin): + if not isinstance(plugin, Plugin): + raise TypeError('Expected Plugin, got: %r' % plugin) + if self.plugins.has_key(name): + raise KeyError('Plugin with name already exists: %r' % name) + self.plugins[name] = plugin + + def unregister_plugin(self, name): + del self.plugins[name] + + def get_plugin(self, name, default=None): + return self.plugins.get(name, default) + + +class Plugin(Configurable): + """Base class for IPython plugins.""" + pass diff --git a/IPython/core/prefilter.py b/IPython/core/prefilter.py index f2b80fe..50a8b67 100755 --- a/IPython/core/prefilter.py +++ b/IPython/core/prefilter.py @@ -31,11 +31,11 @@ import re from IPython.core.alias import AliasManager from IPython.core.autocall import IPyAutocall -from IPython.core.component import Component +from IPython.config.configurable import Configurable from IPython.core.splitinput import split_user_input from IPython.core.page import page -from IPython.utils.traitlets import List, Int, Any, Str, CBool, Bool +from IPython.utils.traitlets import List, Int, Any, Str, CBool, Bool, Instance from IPython.utils.io import Term from IPython.utils.text import make_quoted_expr from IPython.utils.autoattr import auto_attr @@ -169,7 +169,7 @@ class LineInfo(object): #----------------------------------------------------------------------------- -class PrefilterManager(Component): +class PrefilterManager(Configurable): """Main prefilter component. The IPython prefilter is run on all user input before it is run. The @@ -210,19 +210,15 @@ class PrefilterManager(Component): """ multi_line_specials = CBool(True, config=True) + shell = Instance('IPython.core.iplib.InteractiveShellABC') - def __init__(self, parent, config=None): - super(PrefilterManager, self).__init__(parent, config=config) + def __init__(self, shell=None, config=None): + super(PrefilterManager, self).__init__(shell=shell, config=config) + self.shell = shell self.init_transformers() self.init_handlers() self.init_checkers() - @auto_attr - def shell(self): - return Component.get_instances( - root=self.root, - klass='IPython.core.iplib.InteractiveShell')[0] - #------------------------------------------------------------------------- # API for managing transformers #------------------------------------------------------------------------- @@ -231,7 +227,9 @@ class PrefilterManager(Component): """Create the default transformers.""" self._transformers = [] for transformer_cls in _default_transformers: - transformer_cls(self, config=self.config) + transformer_cls( + shell=self.shell, prefilter_manager=self, config=self.config + ) def sort_transformers(self): """Sort the transformers by priority. @@ -265,7 +263,9 @@ class PrefilterManager(Component): """Create the default checkers.""" self._checkers = [] for checker in _default_checkers: - checker(self, config=self.config) + checker( + shell=self.shell, prefilter_manager=self, config=self.config + ) def sort_checkers(self): """Sort the checkers by priority. @@ -300,7 +300,9 @@ class PrefilterManager(Component): self._handlers = {} self._esc_handlers = {} for handler in _default_handlers: - handler(self, config=self.config) + handler( + shell=self.shell, prefilter_manager=self, config=self.config + ) @property def handlers(self): @@ -445,28 +447,22 @@ class PrefilterManager(Component): #----------------------------------------------------------------------------- -class PrefilterTransformer(Component): +class PrefilterTransformer(Configurable): """Transform a line of user input.""" priority = Int(100, config=True) - shell = Any - prefilter_manager = Any + # Transformers don't currently use shell or prefilter_manager, but as we + # move away from checkers and handlers, they will need them. + shell = Instance('IPython.core.iplib.InteractiveShellABC') + prefilter_manager = Instance('IPython.core.prefilter.PrefilterManager') enabled = Bool(True, config=True) - def __init__(self, parent, config=None): - super(PrefilterTransformer, self).__init__(parent, config=config) + def __init__(self, shell=None, prefilter_manager=None, config=None): + super(PrefilterTransformer, self).__init__( + shell=shell, prefilter_manager=prefilter_manager, config=config + ) self.prefilter_manager.register_transformer(self) - @auto_attr - def shell(self): - return Component.get_instances( - root=self.root, - klass='IPython.core.iplib.InteractiveShell')[0] - - @auto_attr - def prefilter_manager(self): - return PrefilterManager.get_instances(root=self.root)[0] - def transform(self, line, continue_prompt): """Transform a line, returning the new one.""" return None @@ -561,28 +557,20 @@ class IPyPromptTransformer(PrefilterTransformer): #----------------------------------------------------------------------------- -class PrefilterChecker(Component): +class PrefilterChecker(Configurable): """Inspect an input line and return a handler for that line.""" priority = Int(100, config=True) - shell = Any - prefilter_manager = Any + shell = Instance('IPython.core.iplib.InteractiveShellABC') + prefilter_manager = Instance('IPython.core.prefilter.PrefilterManager') enabled = Bool(True, config=True) - def __init__(self, parent, config=None): - super(PrefilterChecker, self).__init__(parent, config=config) + def __init__(self, shell=None, prefilter_manager=None, config=None): + super(PrefilterChecker, self).__init__( + shell=shell, prefilter_manager=prefilter_manager, config=config + ) self.prefilter_manager.register_checker(self) - @auto_attr - def shell(self): - return Component.get_instances( - root=self.root, - klass='IPython.core.iplib.InteractiveShell')[0] - - @auto_attr - def prefilter_manager(self): - return PrefilterManager.get_instances(root=self.root)[0] - def check(self, line_info): """Inspect line_info and return a handler instance or None.""" return None @@ -709,16 +697,12 @@ class AliasChecker(PrefilterChecker): priority = Int(800, config=True) - @auto_attr - def alias_manager(self): - return AliasManager.get_instances(root=self.root)[0] - def check(self, line_info): "Check if the initital identifier on the line is an alias." # Note: aliases can not contain '.' head = line_info.ifun.split('.',1)[0] - if line_info.ifun not in self.alias_manager \ - or head not in self.alias_manager \ + if line_info.ifun not in self.shell.alias_manager \ + or head not in self.shell.alias_manager \ or is_shadowed(head, self.shell): return None @@ -766,31 +750,23 @@ class AutocallChecker(PrefilterChecker): #----------------------------------------------------------------------------- -class PrefilterHandler(Component): +class PrefilterHandler(Configurable): handler_name = Str('normal') esc_strings = List([]) - shell = Any - prefilter_manager = Any + shell = Instance('IPython.core.iplib.InteractiveShellABC') + prefilter_manager = Instance('IPython.core.prefilter.PrefilterManager') - def __init__(self, parent, config=None): - super(PrefilterHandler, self).__init__(parent, config=config) + def __init__(self, shell=None, prefilter_manager=None, config=None): + super(PrefilterHandler, self).__init__( + shell=shell, prefilter_manager=prefilter_manager, config=config + ) self.prefilter_manager.register_handler( self.handler_name, self, self.esc_strings ) - @auto_attr - def shell(self): - return Component.get_instances( - root=self.root, - klass='IPython.core.iplib.InteractiveShell')[0] - - @auto_attr - def prefilter_manager(self): - return PrefilterManager.get_instances(root=self.root)[0] - def handle(self, line_info): # print "normal: ", line_info """Handle normal input lines. Use as a template for handlers.""" @@ -827,13 +803,9 @@ class AliasHandler(PrefilterHandler): handler_name = Str('alias') - @auto_attr - def alias_manager(self): - return AliasManager.get_instances(root=self.root)[0] - def handle(self, line_info): """Handle alias input lines. """ - transformed = self.alias_manager.expand_aliases(line_info.ifun,line_info.the_rest) + transformed = self.shell.alias_manager.expand_aliases(line_info.ifun,line_info.the_rest) # pre is needed, because it carries the leading whitespace. Otherwise # aliases won't work in indented sections. line_out = '%sget_ipython().system(%s)' % (line_info.pre_whitespace, @@ -894,7 +866,7 @@ class AutoHandler(PrefilterHandler): esc_strings = List([ESC_PAREN, ESC_QUOTE, ESC_QUOTE2]) def handle(self, line_info): - """Hande lines which can be auto-executed, quoting if requested.""" + """Handle lines which can be auto-executed, quoting if requested.""" line = line_info.line ifun = line_info.ifun the_rest = line_info.the_rest diff --git a/IPython/core/tests/refbug.py b/IPython/core/tests/refbug.py index 6b2ae2b..b049777 100644 --- a/IPython/core/tests/refbug.py +++ b/IPython/core/tests/refbug.py @@ -23,19 +23,25 @@ from IPython.core import ipapi #----------------------------------------------------------------------------- # Globals #----------------------------------------------------------------------------- -ip = ipapi.get() -if not '_refbug_cache' in ip.user_ns: - ip.user_ns['_refbug_cache'] = [] +# This needs to be here because nose and other test runners will import +# this module. Importing this module has potential side effects that we +# want to prevent. +if __name__ == '__main__': + ip = ipapi.get() -aglobal = 'Hello' -def f(): - return aglobal + if not '_refbug_cache' in ip.user_ns: + ip.user_ns['_refbug_cache'] = [] -cache = ip.user_ns['_refbug_cache'] -cache.append(f) -def call_f(): - for func in cache: - print 'lowercased:',func().lower() + aglobal = 'Hello' + def f(): + return aglobal + + cache = ip.user_ns['_refbug_cache'] + cache.append(f) + + def call_f(): + for func in cache: + print 'lowercased:',func().lower() diff --git a/IPython/core/tests/test_component.py b/IPython/core/tests/test_component.py deleted file mode 100644 index 5ffb169..0000000 --- a/IPython/core/tests/test_component.py +++ /dev/null @@ -1,214 +0,0 @@ -#!/usr/bin/env python -# encoding: utf-8 -""" -Tests for IPython.core.component - -Authors: - -* Brian Granger -* Fernando Perez (design help) -""" - -#----------------------------------------------------------------------------- -# Copyright (C) 2008-2009 The IPython Development Team -# -# Distributed under the terms of the BSD License. The full license is in -# the file COPYING, distributed as part of this software. -#----------------------------------------------------------------------------- - -#----------------------------------------------------------------------------- -# Imports -#----------------------------------------------------------------------------- - -from unittest import TestCase - -from IPython.core.component import Component, ComponentError -from IPython.utils.traitlets import ( - TraitError, Int, Float, Str -) -from IPython.config.loader import Config - - -#----------------------------------------------------------------------------- -# Test cases -#----------------------------------------------------------------------------- - - -class TestComponentMeta(TestCase): - - def test_get_instances(self): - class BaseComponent(Component): - pass - c1 = BaseComponent(None) - c2 = BaseComponent(c1) - self.assertEquals(BaseComponent.get_instances(),[c1,c2]) - - def test_get_instances_subclass(self): - class MyComponent(Component): - pass - class MyOtherComponent(MyComponent): - pass - c1 = MyComponent(None) - c2 = MyOtherComponent(c1) - c3 = MyOtherComponent(c2) - self.assertEquals(MyComponent.get_instances(), [c1, c2, c3]) - self.assertEquals(MyOtherComponent.get_instances(), [c2, c3]) - - def test_get_instances_root(self): - class MyComponent(Component): - pass - class MyOtherComponent(MyComponent): - pass - c1 = MyComponent(None) - c2 = MyOtherComponent(c1) - c3 = MyOtherComponent(c2) - c4 = MyComponent(None) - c5 = MyComponent(c4) - self.assertEquals(MyComponent.get_instances(root=c1), [c1, c2, c3]) - self.assertEquals(MyComponent.get_instances(root=c4), [c4, c5]) - - -class TestComponent(TestCase): - - def test_parent_child(self): - c1 = Component(None) - c2 = Component(c1) - c3 = Component(c1) - c4 = Component(c3) - self.assertEquals(c1.parent, None) - self.assertEquals(c2.parent, c1) - self.assertEquals(c3.parent, c1) - self.assertEquals(c4.parent, c3) - self.assertEquals(c1.children, [c2, c3]) - self.assertEquals(c2.children, []) - self.assertEquals(c3.children, [c4]) - self.assertEquals(c4.children, []) - - def test_root(self): - c1 = Component(None) - c2 = Component(c1) - c3 = Component(c1) - c4 = Component(c3) - self.assertEquals(c1.root, c1.root) - self.assertEquals(c2.root, c1) - self.assertEquals(c3.root, c1) - self.assertEquals(c4.root, c1) - - def test_change_parent(self): - c1 = Component(None) - c2 = Component(None) - c3 = Component(c1) - self.assertEquals(c3.root, c1) - self.assertEquals(c3.parent, c1) - self.assertEquals(c1.children,[c3]) - c3.parent = c2 - self.assertEquals(c3.root, c2) - self.assertEquals(c3.parent, c2) - self.assertEquals(c2.children,[c3]) - self.assertEquals(c1.children,[]) - - def test_subclass_parent(self): - c1 = Component(None) - self.assertRaises(TraitError, setattr, c1, 'parent', 10) - - class MyComponent(Component): - pass - c1 = Component(None) - c2 = MyComponent(c1) - self.assertEquals(MyComponent.parent.this_class, Component) - self.assertEquals(c2.parent, c1) - - def test_bad_root(self): - c1 = Component(None) - c2 = Component(None) - c3 = Component(None) - self.assertRaises(ComponentError, setattr, c1, 'root', c2) - c1.parent = c2 - self.assertEquals(c1.root, c2) - self.assertRaises(ComponentError, setattr, c1, 'root', c3) - - -class TestComponentConfig(TestCase): - - def test_default(self): - c1 = Component(None) - c2 = Component(c1) - c3 = Component(c2) - self.assertEquals(c1.config, c2.config) - self.assertEquals(c2.config, c3.config) - - def test_custom(self): - config = Config() - config.foo = 'foo' - config.bar = 'bar' - c1 = Component(None, config=config) - c2 = Component(c1) - c3 = Component(c2) - self.assertEquals(c1.config, config) - self.assertEquals(c2.config, config) - self.assertEquals(c3.config, config) - # Test that copies are not made - self.assert_(c1.config is config) - self.assert_(c2.config is config) - self.assert_(c3.config is config) - self.assert_(c1.config is c2.config) - self.assert_(c2.config is c3.config) - - def test_inheritance(self): - class MyComponent(Component): - a = Int(1, config=True) - b = Float(1.0, config=True) - c = Str('no config') - config = Config() - config.MyComponent.a = 2 - config.MyComponent.b = 2.0 - c1 = MyComponent(None, config=config) - c2 = MyComponent(c1) - self.assertEquals(c1.a, config.MyComponent.a) - self.assertEquals(c1.b, config.MyComponent.b) - self.assertEquals(c2.a, config.MyComponent.a) - self.assertEquals(c2.b, config.MyComponent.b) - c4 = MyComponent(c2, config=Config()) - self.assertEquals(c4.a, 1) - self.assertEquals(c4.b, 1.0) - - def test_parent(self): - class Foo(Component): - a = Int(0, config=True) - b = Str('nope', config=True) - class Bar(Foo): - b = Str('gotit', config=False) - c = Float(config=True) - config = Config() - config.Foo.a = 10 - config.Foo.b = "wow" - config.Bar.b = 'later' - config.Bar.c = 100.0 - f = Foo(None, config=config) - b = Bar(f) - self.assertEquals(f.a, 10) - self.assertEquals(f.b, 'wow') - self.assertEquals(b.b, 'gotit') - self.assertEquals(b.c, 100.0) - - -class TestComponentName(TestCase): - - def test_default(self): - class MyComponent(Component): - pass - c1 = Component(None) - c2 = MyComponent(None) - c3 = Component(c2) - self.assertNotEquals(c1.name, c2.name) - self.assertNotEquals(c1.name, c3.name) - - def test_manual(self): - class MyComponent(Component): - pass - c1 = Component(None, name='foo') - c2 = MyComponent(None, name='bar') - c3 = Component(c2, name='bah') - self.assertEquals(c1.name, 'foo') - self.assertEquals(c2.name, 'bar') - self.assertEquals(c3.name, 'bah') diff --git a/IPython/core/tests/test_plugin.py b/IPython/core/tests/test_plugin.py new file mode 100644 index 0000000..92ff22b --- /dev/null +++ b/IPython/core/tests/test_plugin.py @@ -0,0 +1,46 @@ +"""Tests for plugin.py""" + +#----------------------------------------------------------------------------- +# Imports +#----------------------------------------------------------------------------- + +from unittest import TestCase + +from IPython.core.plugin import Plugin, PluginManager + +#----------------------------------------------------------------------------- +# Tests +#----------------------------------------------------------------------------- + +class FooPlugin(Plugin): + pass + + +class BarPlugin(Plugin): + pass + + +class BadPlugin(object): + pass + + +class PluginTest(TestCase): + + def setUp(self): + self.manager = PluginManager() + + def test_register_get(self): + self.assertEquals(None, self.manager.get_plugin('foo')) + foo = FooPlugin() + self.manager.register_plugin('foo', foo) + self.assertEquals(foo, self.manager.get_plugin('foo')) + bar = BarPlugin() + self.assertRaises(KeyError, self.manager.register_plugin, 'foo', bar) + bad = BadPlugin() + self.assertRaises(TypeError, self.manager.register_plugin, 'bad') + + def test_unregister(self): + foo = FooPlugin() + self.manager.register_plugin('foo', foo) + self.manager.unregister_plugin('foo') + self.assertEquals(None, self.manager.get_plugin('foo')) diff --git a/IPython/extensions/__init__.py b/IPython/extensions/__init__.py index 22e892d..db7f79f 100644 --- a/IPython/extensions/__init__.py +++ b/IPython/extensions/__init__.py @@ -1,13 +1,2 @@ # -*- coding: utf-8 -*- -"""This directory is meant for special-purpose extensions to IPython. - -This can include things which alter the syntax processing stage (see -PhysicalQ_Input for an example of how to do this). - -Any file located here can be called with an 'execfile =' option as - - execfile = extensions/filename.py - -since the IPython directory itself is already part of the search path for -files listed as 'execfile ='. -""" +"""This directory is meant for IPython extensions.""" diff --git a/IPython/extensions/parallelmagic.py b/IPython/extensions/parallelmagic.py index cd3b48f..3f53468 100755 --- a/IPython/extensions/parallelmagic.py +++ b/IPython/extensions/parallelmagic.py @@ -16,8 +16,8 @@ import new -from IPython.core.component import Component -from IPython.utils.traitlets import Bool, Any +from IPython.core.plugin import Plugin +from IPython.utils.traitlets import Bool, Any, Instance from IPython.utils.autoattr import auto_attr from IPython.testing import decorators as testdec @@ -31,28 +31,19 @@ Use activate() on a MultiEngineClient object to activate it for magics. """ -class ParalleMagicComponent(Component): +class ParalleMagic(Plugin): """A component to manage the %result, %px and %autopx magics.""" active_multiengine_client = Any() verbose = Bool(False, config=True) + shell = Instance('IPython.core.iplib.InteractiveShellABC') - def __init__(self, parent, name=None, config=None): - super(ParalleMagicComponent, self).__init__(parent, name=name, config=config) + def __init__(self, shell=None, config=None): + super(ParalleMagic, self).__init__(shell=shell, config=config) self._define_magics() # A flag showing if autopx is activated or not self.autopx = False - # Access other components like this rather than by a regular attribute. - # This won't lookup the InteractiveShell object until it is used and - # then it is cached. This is both efficient and couples this class - # more loosely to InteractiveShell. - @auto_attr - def shell(self): - return Component.get_instances( - root=self.root, - klass='IPython.core.iplib.InteractiveShell')[0] - def _define_magics(self): """Define the magic functions.""" self.shell.define_magic('result', self.magic_result) @@ -204,6 +195,7 @@ def load_ipython_extension(ip): """Load the extension in IPython.""" global _loaded if not _loaded: - prd = ParalleMagicComponent(ip, name='parallel_magic') + plugin = ParalleMagic(shell=ip, config=ip.config) + ip.plugin_manager.register_plugin('parallel_magic', plugin) _loaded = True diff --git a/IPython/extensions/pretty.py b/IPython/extensions/pretty.py index 9484757..8f05b23 100644 --- a/IPython/extensions/pretty.py +++ b/IPython/extensions/pretty.py @@ -37,8 +37,8 @@ by doing:: from IPython.core.error import TryNext from IPython.external import pretty -from IPython.core.component import Component -from IPython.utils.traitlets import Bool, List +from IPython.core.plugin import Plugin +from IPython.utils.traitlets import Bool, List, Instance from IPython.utils.io import Term from IPython.utils.autoattr import auto_attr from IPython.utils.importstring import import_item @@ -51,10 +51,11 @@ from IPython.utils.importstring import import_item _loaded = False -class PrettyResultDisplay(Component): +class PrettyResultDisplay(Plugin): """A component for pretty printing on steroids.""" verbose = Bool(False, config=True) + shell = Instance('IPython.core.iplib.InteractiveShellABC') # A list of (type, func_name), like # [(dict, 'my_dict_printer')] @@ -66,8 +67,8 @@ class PrettyResultDisplay(Component): # The final argument can also be a callable defaults_for_type_by_name = List(default_value=[], config=True) - def __init__(self, parent, name=None, config=None): - super(PrettyResultDisplay, self).__init__(parent, name=name, config=config) + def __init__(self, shell=None, config=None): + super(PrettyResultDisplay, self).__init__(shell=shell, config=config) self._setup_defaults() def _setup_defaults(self): @@ -87,16 +88,6 @@ class PrettyResultDisplay(Component): else: raise TypeError('func_name must be a str or callable, got: %r' % func_name) - # Access other components like this rather than by a regular attribute. - # This won't lookup the InteractiveShell object until it is used and - # then it is cached. This is both efficient and couples this class - # more loosely to InteractiveShell. - @auto_attr - def shell(self): - return Component.get_instances( - root=self.root, - klass='IPython.core.iplib.InteractiveShell')[0] - def __call__(self, otherself, arg): """Uber-pretty-printing display hook. @@ -132,10 +123,10 @@ def load_ipython_extension(ip): """Load the extension in IPython as a hook.""" global _loaded if not _loaded: - prd = PrettyResultDisplay(ip, name='pretty_result_display') - ip.set_hook('result_display', prd, priority=99) + plugin = PrettyResultDisplay(shell=ip, config=ip.config) + ip.set_hook('result_display', plugin, priority=99) _loaded = True - return prd + ip.plugin_manager.register_plugin('pretty_result_display', plugin) def unload_ipython_extension(ip): """Unload the extension.""" diff --git a/IPython/extensions/tests/test_pretty.py b/IPython/extensions/tests/test_pretty.py index b7738cf..c7da78a 100644 --- a/IPython/extensions/tests/test_pretty.py +++ b/IPython/extensions/tests/test_pretty.py @@ -17,8 +17,8 @@ Simple tests for :mod:`IPython.extensions.pretty`. from unittest import TestCase -from IPython.core.component import Component, masquerade_as -from IPython.core.iplib import InteractiveShell +from IPython.config.configurable import Configurable +from IPython.core.iplib import InteractiveShellABC from IPython.extensions import pretty as pretty_ext from IPython.external import pretty from IPython.testing import decorators as dec @@ -29,9 +29,11 @@ from IPython.utils.traitlets import Bool # Tests #----------------------------------------------------------------------------- -class InteractiveShellStub(Component): +class InteractiveShellStub(Configurable): pprint = Bool(True) +InteractiveShellABC.register(InteractiveShellStub) + class A(object): pass @@ -41,12 +43,8 @@ def a_pprinter(o, p, c): class TestPrettyResultDisplay(TestCase): def setUp(self): - self.ip = InteractiveShellStub(None) - # This allows our stub to be retrieved instead of the real - # InteractiveShell - masquerade_as(self.ip, InteractiveShell) - self.prd = pretty_ext.PrettyResultDisplay(self.ip, - name='pretty_result_display') + self.ip = InteractiveShellStub() + self.prd = pretty_ext.PrettyResultDisplay(shell=self.ip, config=None) def test_for_type(self): self.prd.for_type(A, a_pprinter) @@ -77,7 +75,8 @@ a b ip = get_ipython() -prd = ip.load_extension('pretty') +ip.extension_manager.load_extension('pretty') +prd = ip.plugin_manager.get_plugin('pretty_result_display') prd.for_type(A, a_pretty_printer) prd.for_type_by_name(B.__module__, B.__name__, b_pretty_printer) diff --git a/IPython/kernel/clusterdir.py b/IPython/kernel/clusterdir.py index 63174eb..6a7c296 100755 --- a/IPython/kernel/clusterdir.py +++ b/IPython/kernel/clusterdir.py @@ -26,7 +26,7 @@ from twisted.python import log from IPython.config.loader import PyFileConfigLoader from IPython.core.application import Application, BaseAppConfigLoader -from IPython.core.component import Component +from IPython.config.configurable import Configurable from IPython.core.crashhandler import CrashHandler from IPython.core import release from IPython.utils.path import ( @@ -63,7 +63,7 @@ class PIDFileError(Exception): # Class for managing cluster directories #----------------------------------------------------------------------------- -class ClusterDir(Component): +class ClusterDir(Configurable): """An object to manage the cluster directory and its resources. The cluster directory is used by :command:`ipcontroller`, @@ -82,9 +82,8 @@ class ClusterDir(Component): pid_dir = Unicode(u'') location = Unicode(u'') - def __init__(self, location): - super(ClusterDir, self).__init__(None) - self.location = location + def __init__(self, location=u''): + super(ClusterDir, self).__init__(location=location) def _location_changed(self, name, old, new): if not os.path.isdir(new): @@ -166,7 +165,7 @@ class ClusterDir(Component): The full path to the cluster directory. If it does exist, it will be used. If not, it will be created. """ - return ClusterDir(cluster_dir) + return ClusterDir(location=cluster_dir) @classmethod def create_cluster_dir_by_profile(cls, path, profile=u'default'): @@ -183,7 +182,7 @@ class ClusterDir(Component): if not os.path.isdir(path): raise ClusterDirError('Directory not found: %s' % path) cluster_dir = os.path.join(path, u'cluster_' + profile) - return ClusterDir(cluster_dir) + return ClusterDir(location=cluster_dir) @classmethod def find_cluster_dir_by_profile(cls, ipython_dir, profile=u'default'): @@ -216,7 +215,7 @@ class ClusterDir(Component): for p in paths: cluster_dir = os.path.join(p, dirname) if os.path.isdir(cluster_dir): - return ClusterDir(cluster_dir) + return ClusterDir(location=cluster_dir) else: raise ClusterDirError('Cluster directory not found in paths: %s' % dirname) @@ -235,7 +234,7 @@ class ClusterDir(Component): cluster_dir = expand_path(cluster_dir) if not os.path.isdir(cluster_dir): raise ClusterDirError('Cluster directory not found: %s' % cluster_dir) - return ClusterDir(cluster_dir) + return ClusterDir(location=cluster_dir) #----------------------------------------------------------------------------- diff --git a/IPython/kernel/configobjfactory.py b/IPython/kernel/configobjfactory.py index 745cdfb..b16dbfc 100644 --- a/IPython/kernel/configobjfactory.py +++ b/IPython/kernel/configobjfactory.py @@ -18,7 +18,7 @@ configuration system. import zope.interface as zi -from IPython.core.component import Component +from IPython.config.configurable import Configurable #----------------------------------------------------------------------------- # Code @@ -29,7 +29,7 @@ class IConfiguredObjectFactory(zi.Interface): """I am a component that creates a configured object. This class is useful if you want to configure a class that is not a - subclass of :class:`IPython.core.component.Component`. + subclass of :class:`IPython.config.configurable.Configurable`. """ def __init__(config): @@ -39,12 +39,12 @@ class IConfiguredObjectFactory(zi.Interface): """Return an instance of the configured object.""" -class ConfiguredObjectFactory(Component): +class ConfiguredObjectFactory(Configurable): zi.implements(IConfiguredObjectFactory) - def __init__(self, config): - super(ConfiguredObjectFactory, self).__init__(None, config=config) + def __init__(self, config=None): + super(ConfiguredObjectFactory, self).__init__(config=config) def create(self): raise NotImplementedError('create must be implemented in a subclass') @@ -56,24 +56,24 @@ class IAdaptedConfiguredObjectFactory(zi.Interface): This class is useful if you have the adapt an instance and configure it. """ - def __init__(config, adaptee=None): + def __init__(config=None, adaptee=None): """Get ready to adapt adaptee and then configure it using config.""" def create(): """Return an instance of the adapted and configured object.""" -class AdaptedConfiguredObjectFactory(Component): +class AdaptedConfiguredObjectFactory(Configurable): # zi.implements(IAdaptedConfiguredObjectFactory) - def __init__(self, config, adaptee): + def __init__(self, config=None, adaptee=None): # print # print "config pre:", config - super(AdaptedConfiguredObjectFactory, self).__init__(None, config=config) + super(AdaptedConfiguredObjectFactory, self).__init__(config=config) # print # print "config post:", config self.adaptee = adaptee def create(self): - raise NotImplementedError('create must be implemented in a subclass') \ No newline at end of file + raise NotImplementedError('create must be implemented in a subclass') diff --git a/IPython/kernel/fcutil.py b/IPython/kernel/fcutil.py index ec4de2e..b2fb012 100644 --- a/IPython/kernel/fcutil.py +++ b/IPython/kernel/fcutil.py @@ -191,9 +191,9 @@ class FCServiceFactory(AdaptedConfiguredObjectFactory): """This class creates a tub with various services running in it. The basic idea is that :meth:`create` returns a running :class:`Tub` - instance that has a number of Foolscap references registered in it. - This class is a subclass of :class:`IPython.core.component.Component` - so the IPython configuration and component system are used. + instance that has a number of Foolscap references registered in it. This + class is a subclass of :class:`IPython.config.configurable.Configurable` + so the IPython configuration system is used. Attributes ---------- diff --git a/IPython/kernel/launcher.py b/IPython/kernel/launcher.py index 43cf057..fbcc4e9 100644 --- a/IPython/kernel/launcher.py +++ b/IPython/kernel/launcher.py @@ -19,7 +19,7 @@ import os import re import sys -from IPython.core.component import Component +from IPython.config.configurable import Configurable from IPython.external import Itpl from IPython.utils.traitlets import Str, Int, List, Unicode from IPython.utils.path import get_ipython_module_path @@ -77,7 +77,7 @@ class UnknownStatus(LauncherError): pass -class BaseLauncher(Component): +class BaseLauncher(Configurable): """An asbtraction for starting, stopping and signaling a process.""" # In all of the launchers, the work_dir is where child processes will be @@ -89,9 +89,8 @@ class BaseLauncher(Component): # the --work-dir option. work_dir = Unicode(u'') - def __init__(self, work_dir, parent=None, name=None, config=None): - super(BaseLauncher, self).__init__(parent, name, config) - self.work_dir = work_dir + def __init__(self, work_dir=u'', config=None): + super(BaseLauncher, self).__init__(work_dir=work_dir, config=config) self.state = 'before' # can be before, running, after self.stop_deferreds = [] self.start_data = None @@ -265,9 +264,9 @@ class LocalProcessLauncher(BaseLauncher): # spawnProcess. cmd_and_args = List([]) - def __init__(self, work_dir, parent=None, name=None, config=None): + def __init__(self, work_dir=u'', config=None): super(LocalProcessLauncher, self).__init__( - work_dir, parent, name, config + work_dir=work_dir, config=config ) self.process_protocol = None self.start_deferred = None @@ -356,9 +355,9 @@ class LocalEngineSetLauncher(BaseLauncher): ['--log-to-file','--log-level', '40'], config=True ) - def __init__(self, work_dir, parent=None, name=None, config=None): + def __init__(self, work_dir=u'', config=None): super(LocalEngineSetLauncher, self).__init__( - work_dir, parent, name, config + work_dir=work_dir, config=config ) self.launchers = [] @@ -367,7 +366,7 @@ class LocalEngineSetLauncher(BaseLauncher): self.cluster_dir = unicode(cluster_dir) dlist = [] for i in range(n): - el = LocalEngineLauncher(self.work_dir, self) + el = LocalEngineLauncher(work_dir=self.work_dir, config=self.config) # Copy the engine args over to each engine launcher. import copy el.engine_args = copy.deepcopy(self.engine_args) @@ -560,9 +559,9 @@ class WindowsHPCLauncher(BaseLauncher): scheduler = Str('', config=True) job_cmd = Str(find_job_cmd(), config=True) - def __init__(self, work_dir, parent=None, name=None, config=None): + def __init__(self, work_dir=u'', config=None): super(WindowsHPCLauncher, self).__init__( - work_dir, parent, name, config + work_dir=work_dir, config=config ) @property @@ -633,9 +632,9 @@ class WindowsHPCControllerLauncher(WindowsHPCLauncher): extra_args = List([], config=False) def write_job_file(self, n): - job = IPControllerJob(self) + job = IPControllerJob(config=self.config) - t = IPControllerTask(self) + t = IPControllerTask(config=self.config) # The tasks work directory is *not* the actual work directory of # the controller. It is used as the base path for the stdout/stderr # files that the scheduler redirects to. @@ -664,10 +663,10 @@ class WindowsHPCEngineSetLauncher(WindowsHPCLauncher): extra_args = List([], config=False) def write_job_file(self, n): - job = IPEngineSetJob(self) + job = IPEngineSetJob(config=self.config) for i in range(n): - t = IPEngineTask(self) + t = IPEngineTask(config=self.config) # The tasks work directory is *not* the actual work directory of # the engine. It is used as the base path for the stdout/stderr # files that the scheduler redirects to. @@ -725,9 +724,9 @@ class BatchSystemLauncher(BaseLauncher): # The full path to the instantiated batch script. batch_file = Unicode(u'') - def __init__(self, work_dir, parent=None, name=None, config=None): + def __init__(self, work_dir=u'', config=None): super(BatchSystemLauncher, self).__init__( - work_dir, parent, name, config + work_dir=work_dir, config=config ) self.batch_file = os.path.join(self.work_dir, self.batch_file_name) self.context = {} diff --git a/IPython/kernel/winhpcjob.py b/IPython/kernel/winhpcjob.py index c7dc1c9..b862bc8 100644 --- a/IPython/kernel/winhpcjob.py +++ b/IPython/kernel/winhpcjob.py @@ -24,14 +24,14 @@ import uuid from xml.etree import ElementTree as ET -from IPython.core.component import Component +from IPython.config.configurable import Configurable from IPython.utils.traitlets import ( Str, Int, List, Instance, Enum, Bool, CStr ) #----------------------------------------------------------------------------- -# Job and Task Component +# Job and Task classes #----------------------------------------------------------------------------- @@ -74,7 +74,7 @@ def find_username(): return '%s\\%s' % (domain, username) -class WinHPCJob(Component): +class WinHPCJob(Configurable): job_id = Str('') job_name = Str('MyJob', config=True) @@ -165,7 +165,7 @@ class WinHPCJob(Component): self.tasks.append(task) -class WinHPCTask(Component): +class WinHPCTask(Configurable): task_id = Str('') task_name = Str('') @@ -261,8 +261,8 @@ class IPControllerTask(WinHPCTask): unit_type = Str("Core", config=False) work_directory = CStr('', config=False) - def __init__(self, parent, name=None, config=None): - super(IPControllerTask, self).__init__(parent, name, config) + def __init__(self, config=None): + super(IPControllerTask, self).__init__(config=config) the_uuid = uuid.uuid1() self.std_out_file_path = os.path.join('log','ipcontroller-%s.out' % the_uuid) self.std_err_file_path = os.path.join('log','ipcontroller-%s.err' % the_uuid) @@ -289,8 +289,8 @@ class IPEngineTask(WinHPCTask): unit_type = Str("Core", config=False) work_directory = CStr('', config=False) - def __init__(self, parent, name=None, config=None): - super(IPEngineTask,self).__init__(parent, name, config) + def __init__(self, config=None): + super(IPEngineTask,self).__init__(config=config) the_uuid = uuid.uuid1() self.std_out_file_path = os.path.join('log','ipengine-%s.out' % the_uuid) self.std_err_file_path = os.path.join('log','ipengine-%s.err' % the_uuid) diff --git a/IPython/testing/globalipapp.py b/IPython/testing/globalipapp.py index 714554e..efae6e7 100644 --- a/IPython/testing/globalipapp.py +++ b/IPython/testing/globalipapp.py @@ -136,8 +136,8 @@ def start_ipython(): config = tools.default_config() # Create and initialize our test-friendly IPython instance. - shell = iplib.InteractiveShell( - parent=None, config=config, + shell = iplib.InteractiveShell.instance( + config=config, user_ns=ipnsdict(), user_global_ns={} ) diff --git a/IPython/utils/tests/test_traitlets.py b/IPython/utils/tests/test_traitlets.py index fcfc458..79992a2 100755 --- a/IPython/utils/tests/test_traitlets.py +++ b/IPython/utils/tests/test_traitlets.py @@ -360,6 +360,13 @@ class TestHasTraits(TestCase): traits = a.traits(config_key=lambda v: True) self.assertEquals(traits, dict(i=A.i, f=A.f, j=A.j)) + def test_init(self): + class A(HasTraits): + i = Int() + x = Float() + a = A(i=1, x=10.0) + self.assertEquals(a.i, 1) + self.assertEquals(a.x, 10.0) #----------------------------------------------------------------------------- # Tests for specific trait types diff --git a/IPython/utils/traitlets.py b/IPython/utils/traitlets.py index b41f24f..4c7b3f8 100644 --- a/IPython/utils/traitlets.py +++ b/IPython/utils/traitlets.py @@ -372,14 +372,14 @@ class HasTraits(object): __metaclass__ = MetaHasTraits - def __new__(cls, *args, **kw): + def __new__(cls, **kw): # This is needed because in Python 2.6 object.__new__ only accepts # the cls argument. new_meth = super(HasTraits, cls).__new__ if new_meth is object.__new__: inst = new_meth(cls) else: - inst = new_meth(cls, *args, **kw) + inst = new_meth(cls, **kw) inst._trait_values = {} inst._trait_notifiers = {} # Here we tell all the TraitType instances to set their default @@ -398,9 +398,12 @@ class HasTraits(object): return inst - # def __init__(self): - # self._trait_values = {} - # self._trait_notifiers = {} + def __init__(self, **kw): + # Allow trait values to be set using keyword arguments. + # We need to use setattr for this to trigger validation and + # notifications. + for key, value in kw.iteritems(): + setattr(self, key, value) def _notify_trait(self, name, old_value, new_value): @@ -1018,6 +1021,28 @@ class List(Instance): allow_none=allow_none, **metadata) +class Dict(Instance): + """An instance of a Python dict.""" + + def __init__(self, default_value=None, allow_none=True, **metadata): + """Create a dict trait type from a dict. + + The default value is created by doing ``dict(default_value)``, + which creates a copy of the ``default_value``. + """ + if default_value is None: + args = ((),) + elif isinstance(default_value, dict): + args = (default_value,) + elif isinstance(default_value, SequenceTypes): + args = (default_value,) + else: + raise TypeError('default value of Dict was %s' % default_value) + + super(Dict,self).__init__(klass=dict, args=args, + allow_none=allow_none, **metadata) + + class TCPAddress(TraitType): """A trait for an (ip, port) tuple. @@ -1035,4 +1060,3 @@ class TCPAddress(TraitType): if port >= 0 and port <= 65535: return value self.error(obj, value) - diff --git a/docs/Makefile b/docs/Makefile index e6a20f3..2a03b8d 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -39,8 +39,9 @@ pdf: latex all: html pdf -dist: clean all +dist: all mkdir -p dist + rm -rf dist/* ln build/latex/ipython.pdf dist/ cp -al build/html dist/ @echo "Build finished. Final docs are in dist/" @@ -98,3 +99,6 @@ linkcheck: gitwash-update: python ../tools/gitwash_dumper.py source/development ipython cd source/development/gitwash && rename 's/.rst/.txt/' *.rst + +nightly: dist + rsync -avH --delete dist/ ipython:www/doc/nightly \ No newline at end of file diff --git a/docs/autogen_api.py b/docs/autogen_api.py index d098700..7b5842f 100755 --- a/docs/autogen_api.py +++ b/docs/autogen_api.py @@ -27,7 +27,12 @@ if __name__ == '__main__': r'\.config\.default', r'\.config\.profile', r'\.frontend', - r'\.gui' + r'\.gui', + # For now, the zmq code has + # unconditional top-level code so it's + # not import safe. This needs fixing + # soon. + r'\.zmq', ] docwriter.module_skip_patterns += [ r'\.core\.fakemodule', @@ -38,7 +43,7 @@ if __name__ == '__main__': # for each group copied below # AttributeError: __abstractmethods__ - r'\.core\.component', + r'\.config\.configurable', r'\.utils\.traitlets', # AttributeError: __provides__ diff --git a/docs/source/config/extension.txt b/docs/source/config/extension.txt deleted file mode 100644 index e69de29..0000000 --- a/docs/source/config/extension.txt +++ /dev/null diff --git a/docs/source/config/extensions.txt b/docs/source/config/extensions.txt new file mode 100644 index 0000000..d329711 --- /dev/null +++ b/docs/source/config/extensions.txt @@ -0,0 +1,61 @@ +.. _extensions_overview: + +================== +IPython extensions +================== + +Configuration files are just the first level of customization that IPython +supports. The next level is that of extensions. An IPython extension is an +importable Python module that has a a few special function. By defining these +functions, users can customize IPython by accessing the actual runtime objects +of IPython. Here is a sample extension:: + + # myextension.py + + def load_ipython_extension(ipython): + # The ``ipython`` argument is the currently active + # :class:`InteractiveShell` instance that can be used in any way. + # This allows you do to things like register new magics, plugins or + # aliases. + + def unload_ipython_extension(ipython): + # If you want your extension to be unloadable, put that logic here. + +This :func:`load_ipython_extension` function is called after your extension is +imported and the currently active :class:`InteractiveShell` instance is passed +as the only argument. You can do anything you want with IPython at that point. + +The :func:`load_ipython_extension` will be called again is you load or reload +the extension again. It is up to the extension author to add code to manage +that. + +You can put your extension modules anywhere you want, as long as they can be +imported by Python's standard import mechanism. However, to make it easy to +write extensions, you can also put your extensions in +``os.path.join(self.ipython_dir, 'extensions')``. This directory is added to +``sys.path`` automatically. + +Using extensions +================ + +There are two ways you can tell IPython to use your extension: + +1. Listing it in a configuration file. +2. Using the ``%load_ext`` magic function. + +To load an extension called :file:`myextension.py` add the following logic +to your configuration file:: + + c.Global.extensions = [ + 'myextension' + ] + +To load that same extension at runtime, use the ``%load_ext`` magic:: + +.. sourcecode:: ipython + + In [1]: %load_ext myextension + +To summarize, in conjunction with configuration files and profiles, IPython +extensions give you complete and flexible control over your IPython +setup. diff --git a/docs/source/config/index.txt b/docs/source/config/index.txt index e960d71..0ceaf33 100644 --- a/docs/source/config/index.txt +++ b/docs/source/config/index.txt @@ -8,6 +8,8 @@ Configuration and customization :maxdepth: 2 overview.txt + extensions.txt + plugins.txt ipython.txt editors.txt old.txt diff --git a/docs/source/config/ipython.txt b/docs/source/config/ipython.txt index 2b36017..66fc54a 100644 --- a/docs/source/config/ipython.txt +++ b/docs/source/config/ipython.txt @@ -133,4 +133,4 @@ attributes:: c.AliasManager.user_aliases = [ ('la', 'ls -al') - ] \ No newline at end of file + ] diff --git a/docs/source/config/overview.txt b/docs/source/config/overview.txt index 7e7b215..70d0626 100644 --- a/docs/source/config/overview.txt +++ b/docs/source/config/overview.txt @@ -42,29 +42,26 @@ Application: :class:`~IPython.core.application.Application` application is the :command:`ipython` command line program. Each application reads a *single* configuration file and command line options and then produces a master configuration object for the application. This - configuration object is then passed to the components that the application - creates. Components implement the actual logic of the application and know - how to configure themselves given the configuration object. - -Component: :class:`~IPython.core.component.Component` - A component is a regular Python class that serves as a base class for all - main classes in an application. The - :class:`~IPython.core.component.Component` base class is lightweight and - only does two main things. + configuration object is then passed to the configurable objects that the + application creates. These configurable objects implement the actual logic + of the application and know how to configure themselves given the + configuration object. + +Component: :class:`~IPython.config.configurable.Configurable` + A configurable is a regular Python class that serves as a base class for + all main classes in an application. The + :class:`~IPython.config.configurable.Configurable` base class is + lightweight and only does one things. + + This :class:`~IPython.config.configurable.Configurable` is a subclass + of :class:`~IPython.utils.traitlets.HasTraits` that knows how to configure + itself. Class level traits with the metadata ``config=True`` become + values that can be configured from the command line and configuration + files. - First, it keeps track of all instances of itself and provides an - interfaces for querying those instances. This enables components to get - references to other components, even though they are not "nearby" in the - runtime object graph. - - Second, it declares what class attributes are configurable and specifies - the default types and values of those attributes. This information is used - to automatically configure instances given the applications configuration - object. - - Developers create :class:`~IPython.core.component.Component` subclasses - that implement all of the logic in the application. Each of these - subclasses has its own configuration information that controls how + Developers create :class:`~IPython.config.configurable.Configurable` + subclasses that implement all of the logic in the application. Each of + these subclasses has its own configuration information that controls how instances are created. Having described these main concepts, we can now state the main idea in our @@ -73,7 +70,9 @@ attributes to be controlled on a class by class basis*. Thus all instances of a given class are configured in the same way. Furthermore, if two instances need to be configured differently, they need to be instances of two different classes. While this model may seem a bit restrictive, we have found that it -expresses most things that need to be configured extremely well. +expresses most things that need to be configured extremely well. However, it +is possible to create two instances of the same class that have different +trait values. This is done by overriding the configuration. Now, we show what our configuration objects and files look like. @@ -98,33 +97,34 @@ attributes on it. All you have to know is: * The type of each attribute. The answers to these two questions are provided by the various -:class:`~IPython.core.component.Component` subclasses that an application -uses. Let's look at how this would work for a simple component subclass:: +:class:`~IPython.config.configurable.Configurable` subclasses that an +application uses. Let's look at how this would work for a simple component +subclass:: # Sample component that can be configured. - from IPython.core.component import Component + from IPython.config.configurable import Configurable from IPython.utils.traitlets import Int, Float, Str, Bool - class MyComponent(Component): + class MyClass(Configurable): name = Str('defaultname', config=True) ranking = Int(0, config=True) value = Float(99.0) # The rest of the class implementation would go here.. -In this example, we see that :class:`MyComponent` has three attributes, two +In this example, we see that :class:`MyClass` has three attributes, two of whom (``name``, ``ranking``) can be configured. All of the attributes -are given types and default values. If a :class:`MyComponent` is instantiated, +are given types and default values. If a :class:`MyClass` is instantiated, but not configured, these default values will be used. But let's see how to configure this class in a configuration file:: # Sample config file c = get_config() - c.MyComponent.name = 'coolname' - c.MyComponent.ranking = 10 + c.MyClass.name = 'coolname' + c.MyClass.ranking = 10 After this configuration file is loaded, the values set in it will override -the class defaults anytime a :class:`MyComponent` is created. Furthermore, +the class defaults anytime a :class:`MyClass` is created. Furthermore, these attributes will be type checked and validated anytime they are set. This type checking is handled by the :mod:`IPython.utils.traitlets` module, which provides the :class:`Str`, :class:`Int` and :class:`Float` types. In @@ -133,7 +133,7 @@ traitlets for a number of other types. .. note:: - Underneath the hood, the :class:`Component` base class is a subclass of + Underneath the hood, the :class:`Configurable` base class is a subclass of :class:`IPython.utils.traitlets.HasTraits`. The :mod:`IPython.utils.traitlets` module is a lightweight version of :mod:`enthought.traits`. Our implementation is a pure Python subset @@ -157,7 +157,7 @@ attribute of ``c`` is not the actual class, but instead is another .. note:: - The careful reader may wonder how the ``ClassName`` (``MyComponent`` in + The careful reader may wonder how the ``ClassName`` (``MyClass`` in the above example) attribute of the configuration object ``c`` gets created. These attributes are created on the fly by the :class:`~IPython.config.loader.Config` instance, using a simple naming @@ -165,8 +165,7 @@ attribute of ``c`` is not the actual class, but instead is another instance whose name begins with an uppercase character is assumed to be a sub-configuration and a new empty :class:`~IPython.config.loader.Config` instance is dynamically created for that attribute. This allows deeply - hierarchical information created easily (``c.Foo.Bar.value``) on the - fly. + hierarchical information created easily (``c.Foo.Bar.value``) on the fly. Configuration files inheritance =============================== @@ -179,8 +178,8 @@ example that loads all of the values from the file :file:`base_config.py`:: # base_config.py c = get_config() - c.MyComponent.name = 'coolname' - c.MyComponent.ranking = 100 + c.MyClass.name = 'coolname' + c.MyClass.ranking = 100 into the configuration file :file:`main_config.py`:: @@ -191,7 +190,7 @@ into the configuration file :file:`main_config.py`:: load_subconfig('base_config.py') # Now override one of the values - c.MyComponent.name = 'bettername' + c.MyClass.name = 'bettername' In a situation like this the :func:`load_subconfig` makes sure that the search path for sub-configuration files is inherited from that of the parent. @@ -205,10 +204,10 @@ There is another aspect of configuration where inheritance comes into play. Sometimes, your classes will have an inheritance hierarchy that you want to be reflected in the configuration system. Here is a simple example:: - from IPython.core.component import Component + from IPython.config.configurable import Configurable from IPython.utils.traitlets import Int, Float, Str, Bool - class Foo(Component): + class Foo(Configurable): name = Str('fooname', config=True) value = Float(100.0, config=True) @@ -328,4 +327,3 @@ Here are the main requirements we wanted our configuration system to have: dynamic language and you don't always know everything that needs to be configured when a program starts. - diff --git a/docs/source/config/plugins.txt b/docs/source/config/plugins.txt new file mode 100644 index 0000000..c0aa1ae --- /dev/null +++ b/docs/source/config/plugins.txt @@ -0,0 +1,23 @@ +.. _plugins_overview: + +=============== +IPython plugins +=============== + +IPython has a plugin mechanism that allows users to create new and custom +runtime components for IPython. Plugins are different from extensions: + +* Extensions are used to load plugins. +* Extensions are a more advanced configuration system that gives you access + to the running IPython instance. +* Plugins add entirely new capabilities to IPython. +* Plugins are traited and configurable. + +At this point, our plugin system is brand new and the documentation is +minimal. If you are interested in creating a new plugin, see the following +files: + +* :file:`IPython/extensions/parallemagic.py` +* :file:`IPython/extensions/pretty.` + +As well as our documentation on the configuration system and extensions. diff --git a/docs/source/development/core_design.txt b/docs/source/development/core_design.txt deleted file mode 100644 index 423de37..0000000 --- a/docs/source/development/core_design.txt +++ /dev/null @@ -1,286 +0,0 @@ -======================================== - Design proposal for mod:`IPython.core` -======================================== - -Currently mod:`IPython.core` is not well suited for use in GUI applications. -The purpose of this document is to describe a design that will resolve this -limitation. - -Process and thread model -======================== - -The design described here is based on a two process model. These two processes -are: - -1. The IPython engine/kernel. This process contains the user's namespace and - is responsible for executing user code. If user code uses - :mod:`enthought.traits` or uses a GUI toolkit to perform plotting, the GUI - event loop will run in this process. - -2. The GUI application. The user facing GUI application will run in a second - process that communicates directly with the IPython engine using - asynchronous messaging. The GUI application will not execute any user code. - The canonical example of a GUI application that talks to the IPython - engine, would be a GUI based IPython terminal. However, the GUI application - could provide a more sophisticated interface such as a notebook. - -We now describe the threading model of the IPython engine. Two threads will be -used to implement the IPython engine: a main thread that executes user code -and a networking thread that communicates with the outside world. This -specific design is required by a number of different factors. - -First, The IPython engine must run the GUI event loop if the user wants to -perform interactive plotting. Because of the design of most GUIs, this means -that the user code (which will make GUI calls) must live in the main thread. - -Second, networking code in the engine (Twisted or otherwise) must be able to -communicate with the outside world while user code runs. An example would be -if user code does the following:: - - import time - for i in range(10): - print i - time.sleep(2) - -We would like to result of each ``print i`` to be seen by the GUI application -before the entire code block completes. We call this asynchronous printing. -For this to be possible, the networking code has to be able to be able to -communicate the current value of ``sys.stdout`` to the GUI application while -user code is run. Another example is using :mod:`IPython.kernel.client` in -user code to perform a parallel computation by talking to an IPython -controller and a set of engines (these engines are separate from the one we -are discussing here). This module requires the Twisted event loop to be run in -a different thread than user code. - -For the GUI application, threads are optional. However, the GUI application -does need to be able to perform network communications asynchronously (without -blocking the GUI itself). With this in mind, there are two options: - -* Use Twisted (or another non-blocking socket library) in the same thread as - the GUI event loop. - -* Don't use Twisted, but instead run networking code in the GUI application - using blocking sockets in threads. This would require the usage of polling - and queues to manage the networking in the GUI application. - -Thus, for the GUI application, there is a choice between non-blocking sockets -(Twisted) or threads. - -Asynchronous messaging -====================== - -The GUI application will use asynchronous message queues to communicate with -the networking thread of the engine. Because this communication will typically -happen over localhost, a simple, one way, network protocol like XML-RPC or -JSON-RPC can be used to implement this messaging. These options will also make -it easy to implement the required networking in the GUI application using the -standard library. In applications where secure communications are required, -Twisted and Foolscap will probably be the best way to go for now, but HTTP is -also an option. - -There is some flexibility as to where the message queues are located. One -option is that we could create a third process (like the IPython controller) -that only manages the message queues. This is attractive, but does require -an additional process. - -Using this communication channel, the GUI application and kernel/engine will -be able to send messages back and forth. For the most part, these messages -will have a request/reply form, but it will be possible for the kernel/engine -to send multiple replies for a single request. - -The GUI application will use these messages to control the engine/kernel. -Examples of the types of things that will be possible are: - -* Pass code (as a string) to be executed by the engine in the user's namespace - as a string. - -* Get the current value of stdout and stderr. - -* Get the ``repr`` of an object returned (Out []:). - -* Pass a string to the engine to be completed when the GUI application - receives a tab completion event. - -* Get a list of all variable names in the user's namespace. - -The in memory format of a message should be a Python dictionary, as this -will be easy to serialize using virtually any network protocol. The -message dict should only contain basic types, such as strings, floats, -ints, lists, tuples and other dicts. - -Each message will have a unique id and will probably be determined by the -messaging system and returned when something is queued in the message -system. This unique id will be used to pair replies with requests. - -Each message should have a header of key value pairs that can be introspected -by the message system and a body, or payload, that is opaque. The queues -themselves will be purpose agnostic, so the purpose of the message will have -to be encoded in the message itself. While we are getting started, we -probably don't need to distinguish between the header and body. - -Here are some examples:: - - m1 = dict( - method='execute', - id=24, # added by the message system - parent=None # not a reply, - source_code='a=my_func()' - ) - -This single message could generate a number of reply messages:: - - m2 = dict( - method='stdout' - id=25, # my id, added by the message system - parent_id=24, # The message id of the request - value='This was printed by my_func()' - ) - - m3 = dict( - method='stdout' - id=26, # my id, added by the message system - parent_id=24, # The message id of the request - value='This too was printed by my_func() at a later time.' - ) - - m4 = dict( - method='execute_finished', - id=27, - parent_id=24 - # not sure what else should come back with this message, - # but we will need a way for the GUI app to tell that an execute - # is done. - ) - -We should probably use flags for the method and other purposes: - -EXECUTE='0' -EXECUTE_REPLY='1' - -This will keep out network traffic down and enable us to easily change the -actual value that is sent. - -Engine details -============== - -As discussed above, the engine will consist of two threads: a main thread and -a networking thread. These two threads will communicate using a pair of -queues: one for data and requests passing to the main thread (the main -thread's "input queue") and another for data and requests passing out of the -main thread (the main thread's "output queue"). Both threads will have an -event loop that will enqueue elements on one queue and dequeue elements on the -other queue. - -The event loop of the main thread will be of a different nature depending on -if the user wants to perform interactive plotting. If they do want to perform -interactive plotting, the main threads event loop will simply be the GUI event -loop. In that case, GUI timers will be used to monitor the main threads input -queue. When elements appear on that queue, the main thread will respond -appropriately. For example, if the queue contains an element that consists of -user code to execute, the main thread will call the appropriate method of its -IPython instance. If the user does not want to perform interactive plotting, -the main thread will have a simpler event loop that will simply block on the -input queue. When something appears on that queue, the main thread will awake -and handle the request. - -The event loop of the networking thread will typically be the Twisted event -loop. While it is possible to implement the engine's networking without using -Twisted, at this point, Twisted provides the best solution. Note that the GUI -application does not need to use Twisted in this case. The Twisted event loop -will contain an XML-RPC or JSON-RPC server that takes requests over the -network and handles those requests by enqueing elements on the main thread's -input queue or dequeing elements on the main thread's output queue. - -Because of the asynchronous nature of the network communication, a single -input and output queue will be used to handle the interaction with the main -thread. It is also possible to use multiple queues to isolate the different -types of requests, but our feeling is that this is more complicated than it -needs to be. - -One of the main issues is how stdout/stderr will be handled. Our idea is to -replace sys.stdout/sys.stderr by custom classes that will immediately write -data to the main thread's output queue when user code writes to these streams -(by doing print). Once on the main thread's output queue, the networking -thread will make the data available to the GUI application over the network. - -One unavoidable limitation in this design is that if user code does a print -and then enters non-GIL-releasing extension code, the networking thread will -go silent until the GIL is again released. During this time, the networking -thread will not be able to process the GUI application's requests of the -engine. Thus, the values of stdout/stderr will be unavailable during this -time. This goes beyond stdout/stderr, however. Anytime the main thread is -holding the GIL, the networking thread will go silent and be unable to handle -requests. - -GUI Application details -======================= - -The GUI application will also have two threads. While this is not a strict -requirement, it probably makes sense and is a good place to start. The main -thread will be the GUI tread. The other thread will be a networking thread and -will handle the messages that are sent to and from the engine process. - -Like the engine, we will use two queues to control the flow of messages -between the main thread and networking thread. One of these queues will be -used for messages sent from the GUI application to the engine. When the GUI -application needs to send a message to the engine, it will simply enque the -appropriate message on this queue. The networking thread will watch this queue -and forward messages to the engine using an appropriate network protocol. - -The other queue will be used for incoming messages from the engine. The -networking thread will poll for incoming messages from the engine. When it -receives any message, it will simply put that message on this other queue. The -GUI application will periodically see if there are any messages on this queue -and if there are it will handle them. - -The GUI application must be prepared to handle any incoming message at any -time. Due to a variety of reasons, the one or more reply messages associated -with a request, may appear at any time in the future and possible in different -orders. It is also possible that a reply might not appear. An example of this -would be a request for a tab completion event. If the engine is busy, it won't -be possible to fulfill the request for a while. While the tab completion -request will eventually be handled, the GUI application has to be prepared to -abandon waiting for the reply if the user moves on or a certain timeout -expires. - -Prototype details -================= - -With this design, it should be possible to develop a relatively complete GUI -application, while using a mock engine. This prototype should use the two -process design described above, but instead of making actual network calls, -the network thread of the GUI application should have an object that fakes the -network traffic. This mock object will consume messages off of one queue, -pause for a short while (to model network and other latencies) and then place -reply messages on the other queue. - -This simple design will allow us to determine exactly what the message types -and formats should be as well as how the GUI application should interact with -the two message queues. Note, it is not required that the mock object actually -be able to execute Python code or actually complete strings in the users -namespace. All of these things can simply be faked. This will also help us to -understand what the interface needs to look like that handles the network -traffic. This will also help us to understand the design of the engine better. - -The GUI application should be developed using IPython's component, application -and configuration system. It may take some work to see what the best way of -integrating these things with PyQt are. - -After this stage is done, we can move onto creating a real IPython engine for -the GUI application to communicate with. This will likely be more work that -the GUI application itself, but having a working GUI application will make it -*much* easier to design and implement the engine. - -We also might want to introduce a third process into the mix. Basically, this -would be a central messaging hub that both the engine and GUI application -would use to send and retrieve messages. This is not required, but it might be -a really good idea. - -Also, I have some ideas on the best way to handle notebook saving and -persistence. - -Refactoring of IPython.core -=========================== - -We need to go through IPython.core and describe what specifically needs to be -done. diff --git a/docs/source/development/messaging.txt b/docs/source/development/messaging.txt index 408212e..e2c0dc4 100644 --- a/docs/source/development/messaging.txt +++ b/docs/source/development/messaging.txt @@ -1,106 +1,579 @@ -===================== -Message Specification -===================== +====================== + Messaging in IPython +====================== -Note: not all of these have yet been fully fleshed out, but the key ones are, -see kernel and frontend files for actual implementation details. -General Message Format +Introduction +============ + +This document explains the basic communications design and messaging +specification for how the various IPython objects interact over a network +transport. The current implementation uses the ZeroMQ_ library for messaging +within and between hosts. + +.. Note:: + + This document should be considered the authoritative description of the + IPython messaging protocol, and all developers are strongly encouraged to + keep it updated as the implementation evolves, so that we have a single + common reference for all protocol details. + +The basic design is explained in the following diagram: + +.. image:: frontend-kernel.png + :width: 450px + :alt: IPython kernel/frontend messaging architecture. + :align: center + :target: ../_images/frontend-kernel.png + +A single kernel can be simultaneously connected to one or more frontends. The +kernel has three sockets that serve the following functions: + +1. REQ: this socket is connected to a *single* frontend at a time, and it allows + the kernel to request input from a frontend when :func:`raw_input` is called. + The frontend holding the matching REP socket acts as a 'virtual keyboard' + for the kernel while this communication is happening (illustrated in the + figure by the black outline around the central keyboard). In practice, + frontends may display such kernel requests using a special input widget or + otherwise indicating that the user is to type input for the kernel instead + of normal commands in the frontend. + +2. XREP: this single sockets allows multiple incoming connections from + frontends, and this is the socket where requests for code execution, object + information, prompts, etc. are made to the kernel by any frontend. The + communication on this socket is a sequence of request/reply actions from + each frontend and the kernel. + +3. PUB: this socket is the 'broadcast channel' where the kernel publishes all + side effects (stdout, stderr, etc.) as well as the requests coming from any + client over the XREP socket and its own requests on the REP socket. There + are a number of actions in Python which generate side effects: :func:`print` + writes to ``sys.stdout``, errors generate tracebacks, etc. Additionally, in + a multi-client scenario, we want all frontends to be able to know what each + other has sent to the kernel (this can be useful in collaborative scenarios, + for example). This socket allows both side effects and the information + about communications taking place with one client over the XREQ/XREP channel + to be made available to all clients in a uniform manner. + + All messages are tagged with enough information (details below) for clients + to know which messages come from their own interaction with the kernel and + which ones are from other clients, so they can display each type + appropriately. + +The actual format of the messages allowed on each of these channels is +specified below. Messages are dicts of dicts with string keys and values that +are reasonably representable in JSON. Our current implementation uses JSON +explicitly as its message format, but this shouldn't be considered a permanent +feature. As we've discovered that JSON has non-trivial performance issues due +to excessive copying, we may in the future move to a pure pickle-based raw +message format. However, it should be possible to easily convert from the raw +objects to JSON, since we may have non-python clients (e.g. a web frontend). +As long as it's easy to make a JSON version of the objects that is a faithful +representation of all the data, we can communicate with such clients. + +.. Note:: + + Not all of these have yet been fully fleshed out, but the key ones are, see + kernel and frontend files for actual implementation details. + + +Python functional API ===================== -General message format:: +As messages are dicts, they map naturally to a ``func(**kw)`` call form. We +should develop, at a few key points, functional forms of all the requests that +take arguments in this manner and automatically construct the necessary dict +for sending. + + +General Message Format +====================== +All messages send or received by any IPython process should have the following +generic structure:: + { - header : { 'msg_id' : 10, # start with 0 - 'username' : 'name', + # The message header contains a pair of unique identifiers for the + # originating session and the actual message id, in addition to the + # username for the process that generated the message. This is useful in + # collaborative settings where multiple users may be interacting with the + # same kernel simultaneously, so that frontends can label the various + # messages in a meaningful way. + 'header' : { 'msg_id' : uuid, + 'username' : str, 'session' : uuid - }, - parent_header : dict, - msg_type : 'string_message_type', - content : blackbox_dict , # Must be a dict + }, + + # In a chain of messages, the header from the parent is copied so that + # clients can track where messages come from. + 'parent_header' : dict, + + # All recognized message type strings are listed below. + 'msg_type' : str, + + # The actual content of the message must be a dict, whose structure + # depends on the message type.x + 'content' : dict, } -Side effect: (PUB/SUB) -====================== +For each message type, the actual content will differ and all existing message +types are specified in what follows of this document. -# msg_type = 'stream':: + +Messages on the XREP/XREQ socket +================================ + +.. _execute: + +Execute +------- + +The execution request contains a single string, but this may be a multiline +string. The kernel is responsible for splitting this into possibly more than +one block and deciding whether to compile these in 'single' or 'exec' mode. +We're still sorting out this policy. The current inputsplitter is capable of +splitting the input for blocks that can all be run as 'single', but in the long +run it may prove cleaner to only use 'single' mode for truly single-line +inputs, and run all multiline input in 'exec' mode. This would preserve the +natural behavior of single-line inputs while allowing long cells to behave more +likea a script. This design will be refined as we complete the implementation. + +Message type: ``execute_request``:: content = { - name : 'stdout', - data : 'blob', + # Source code to be executed by the kernel, one or more lines. + 'code' : str, + + # A boolean flag which, if True, signals the kernel to execute this + # code as quietly as possible. This means that the kernel will compile + # the code with 'exec' instead of 'single' (so sys.displayhook will not + # fire), and will *not*: + # - broadcast exceptions on the PUB socket + # - do any logging + # - populate any history + # The default is False. + 'silent' : bool, } -# msg_type = 'pyin':: +Upon execution, the kernel *always* sends a reply, with a status code +indicating what happened and additional data depending on the outcome. + +Message type: ``execute_reply``:: content = { - code = 'x=1', + # One of: 'ok' OR 'error' OR 'abort' + 'status' : str, + + # Any additional data depends on status value } -# msg_type = 'pyout':: +When status is 'ok', the following extra fields are present:: + + { + # This has the same structure as the output of a prompt request, but is + # for the client to set up the *next* prompt (with identical limitations + # to a prompt request) + 'next_prompt' : { + 'prompt_string' : str, + 'prompt_number' : int, + }, + + # The prompt number of the actual execution for this code, which may be + # different from the one used when the code was typed, which was the + # 'next_prompt' field of the *previous* request. They will differ in the + # case where there is more than one client talking simultaneously to a + # kernel, since the numbers can go out of sync. GUI clients can use this + # to correct the previously written number in-place, terminal ones may + # re-print a corrected one if desired. + 'prompt_number' : int, + + # The kernel will often transform the input provided to it. This + # contains the transformed code, which is what was actually executed. + 'transformed_code' : str, + + # The execution payload is a dict with string keys that may have been + # produced by the code being executed. It is retrieved by the kernel at + # the end of the execution and sent back to the front end, which can take + # action on it as needed. See main text for further details. + 'payload' : dict, + } + +.. admonition:: Execution payloads + + The notion of an 'execution payload' is different from a return value of a + given set of code, which normally is just displayed on the pyout stream + through the PUB socket. The idea of a payload is to allow special types of + code, typically magics, to populate a data container in the IPython kernel + that will be shipped back to the caller via this channel. The kernel will + have an API for this, probably something along the lines of:: + + ip.exec_payload_add(key, value) + + though this API is still in the design stages. The data returned in this + payload will allow frontends to present special views of what just happened. + + +When status is 'error', the following extra fields are present:: + + { + 'exc_name' : str, # Exception name, as a string + 'exc_value' : str, # Exception value, as a string + + # The traceback will contain a list of frames, represented each as a + # string. For now we'll stick to the existing design of ultraTB, which + # controls exception level of detail statefully. But eventually we'll + # want to grow into a model where more information is collected and + # packed into the traceback object, with clients deciding how little or + # how much of it to unpack. But for now, let's start with a simple list + # of strings, since that requires only minimal changes to ultratb as + # written. + 'traceback' : list, + } + + +When status is 'abort', there are for now no additional data fields. This +happens when the kernel was interrupted by a signal. + + +Prompt +------ + +A simple request for a current prompt string. + +Message type: ``prompt_request``:: + + content = {} + +In the reply, the prompt string comes back with the prompt number placeholder +*unevaluated*. The message format is: + +Message type: ``prompt_reply``:: + + content = { + 'prompt_string' : str, + 'prompt_number' : int, + } + +Clients can produce a prompt with ``prompt_string.format(prompt_number)``, but +they should be aware that the actual prompt number for that input could change +later, in the case where multiple clients are interacting with a single +kernel. + + +Object information +------------------ + +One of IPython's most used capabilities is the introspection of Python objects +in the user's namespace, typically invoked via the ``?`` and ``??`` characters +(which in reality are shorthands for the ``%pinfo`` magic). This is used often +enough that it warrants an explicit message type, especially because frontends +may want to get object information in response to user keystrokes (like Tab or +F1) besides from the user explicitly typing code like ``x??``. + +Message type: ``object_info_request``:: content = { - data = 'repr(obj)', - prompt_number = 10 + # The (possibly dotted) name of the object to be searched in all + # relevant namespaces + 'name' : str, + + # The level of detail desired. The default (0) is equivalent to typing + # 'x?' at the prompt, 1 is equivalent to 'x??'. + 'detail_level' : int, } -# msg_type = 'pyerr':: +The returned information will be a dictionary with keys very similar to the +field names that IPython prints at the terminal. + +Message type: ``object_info_reply``:: content = { - traceback : 'full traceback', - exc_type : 'TypeError', - exc_value : 'msg' + # Flags for magics and system aliases + 'ismagic' : bool, + 'isalias' : bool, + + # The name of the namespace where the object was found ('builtin', + # 'magics', 'alias', 'interactive', etc.) + 'namespace' : str, + + # The type name will be type.__name__ for normal Python objects, but it + # can also be a string like 'Magic function' or 'System alias' + 'type_name' : str, + + 'string_form' : str, + + # For objects with a __class__ attribute this will be set + 'base_class' : str, + + # For objects with a __len__ attribute this will be set + 'length' : int, + + # If the object is a function, class or method whose file we can find, + # we give its full path + 'file' : str, + + # For pure Python callable objects, we can reconstruct the object + # definition line which provides its call signature + 'definition' : str, + + # For instances, provide the constructor signature (the definition of + # the __init__ method): + 'init_definition' : str, + + # Docstrings: for any object (function, method, module, package) with a + # docstring, we show it. But in addition, we may provide additional + # docstrings. For example, for instances we will show the constructor + # and class docstrings as well, if available. + 'docstring' : str, + + # For instances, provide the constructor and class docstrings + 'init_docstring' : str, + 'class_docstring' : str, + + # If detail_level was 1, we also try to find the source code that + # defines the object, if possible. The string 'None' will indicate + # that no source was found. + 'source' : str, } -# msg_type = 'file': + +Complete +-------- + +Message type: ``complete_request``:: + content = { - path = 'cool.jpg', - data : 'blob' + # The text to be completed, such as 'a.is' + 'text' : str, + + # The full line, such as 'print a.is'. This allows completers to + # make decisions that may require information about more than just the + # current word. + 'line' : str, } -Request/Reply -============= +Message type: ``complete_reply``:: -Execute + content = { + # The list of all matches to the completion request, such as + # ['a.isalnum', 'a.isalpha'] for the above example. + 'matches' : list + } + + +History ------- -Request: +For clients to explicitly request history from a kernel. The kernel has all +the actual execution history stored in a single location, so clients can +request it from the kernel when needed. -# msg_type = 'execute_request':: +Message type: ``history_request``:: content = { - code : 'a = 10', + + # If true, also return output history in the resulting dict. + 'output' : bool, + + # This parameter can be one of: A number, a pair of numbers, 'all' + # If not given, last 40 are returned. + # - number n: return the last n entries. + # - pair n1, n2: return entries in the range(n1, n2). + # - 'all': return all history + 'range' : n or (n1, n2) or 'all', + + # If a filter is given, it is treated as a regular expression and only + # matching entries are returned. re.search() is used to find matches. + 'filter' : str, } -Reply: +Message type: ``history_reply``:: + + content = { + # A list of (number, input) pairs + 'input' : list, + + # A list of (number, output) pairs + 'output' : list, + } -# msg_type = 'execute_reply':: + +Messages on the PUB/SUB socket +============================== + +Streams (stdout, stderr, etc) +------------------------------ + +Message type: ``stream``:: content = { - 'status' : 'ok' OR 'error' OR 'abort' - # data depends on status value + # The name of the stream is one of 'stdin', 'stdout', 'stderr' + 'name' : str, + + # The data is an arbitrary string to be written to that stream + 'data' : str, } -Complete --------- +When a kernel receives a raw_input call, it should also broadcast it on the pub +socket with the names 'stdin' and 'stdin_reply'. This will allow other clients +to monitor/display kernel interactions and possibly replay them to their user +or otherwise expose them. -# msg_type = 'complete_request':: +Python inputs +------------- + +These messages are the re-broadcast of the ``execute_request``. + +Message type: ``pyin``:: content = { - text : 'a.f', # complete on this - line : 'print a.f' # full line + # Source code to be executed, one or more lines + 'code' : str } -# msg_type = 'complete_reply':: +Python outputs +-------------- + +When Python produces output from code that has been compiled in with the +'single' flag to :func:`compile`, any expression that produces a value (such as +``1+1``) is passed to ``sys.displayhook``, which is a callable that can do with +this value whatever it wants. The default behavior of ``sys.displayhook`` in +the Python interactive prompt is to print to ``sys.stdout`` the :func:`repr` of +the value as long as it is not ``None`` (which isn't printed at all). In our +case, the kernel instantiates as ``sys.displayhook`` an object which has +similar behavior, but which instead of printing to stdout, broadcasts these +values as ``pyout`` messages for clients to display appropriately. + +Message type: ``pyout``:: content = { - matches : ['a.foo', 'a.bar'] + # The data is typically the repr() of the object. + 'data' : str, + + # The prompt number for this execution is also provided so that clients + # can display it, since IPython automatically creates variables called + # _N (for prompt N). + 'prompt_number' : int, } + +Python errors +------------- -Control -------- +When an error occurs during code execution -# msg_type = 'heartbeat':: +Message type: ``pyerr``:: content = { - # XXX - unfinished + # Similar content to the execute_reply messages for the 'error' case, + # except the 'status' field is omitted. } + +Kernel crashes +-------------- + +When the kernel has an unexpected exception, caught by the last-resort +sys.excepthook, we should broadcast the crash handler's output before exiting. +This will allow clients to notice that a kernel died, inform the user and +propose further actions. + +Message type: ``crash``:: + + content = { + # Similarly to the 'error' case for execute_reply messages, this will + # contain exc_name, exc_type and traceback fields. + + # An additional field with supplementary information such as where to + # send the crash message + 'info' : str, + } + + +Future ideas +------------ + +Other potential message types, currently unimplemented, listed below as ideas. + +Message type: ``file``:: + + content = { + 'path' : 'cool.jpg', + 'mimetype' : str, + 'data' : str, + } + + +Messages on the REQ/REP socket +============================== + +This is a socket that goes in the opposite direction: from the kernel to a +*single* frontend, and its purpose is to allow ``raw_input`` and similar +operations that read from ``sys.stdin`` on the kernel to be fulfilled by the +client. For now we will keep these messages as simple as possible, since they +basically only mean to convey the ``raw_input(prompt)`` call. + +Message type: ``input_request``:: + + content = { 'prompt' : str } + +Message type: ``input_reply``:: + + content = { 'value' : str } + +.. Note:: + + We do not explicitly try to forward the raw ``sys.stdin`` object, because in + practice the kernel should behave like an interactive program. When a + program is opened on the console, the keyboard effectively takes over the + ``stdin`` file descriptor, and it can't be used for raw reading anymore. + Since the IPython kernel effectively behaves like a console program (albeit + one whose "keyboard" is actually living in a separate process and + transported over the zmq connection), raw ``stdin`` isn't expected to be + available. + + +Heartbeat for kernels +===================== + +Initially we had considered using messages like those above over ZMQ for a +kernel 'heartbeat' (a way to detect quickly and reliably whether a kernel is +alive at all, even if it may be busy executing user code). But this has the +problem that if the kernel is locked inside extension code, it wouldn't execute +the python heartbeat code. But it turns out that we can implement a basic +heartbeat with pure ZMQ, without using any Python messaging at all. + +The monitor sends out a single zmq message (right now, it is a str of the +monitor's lifetime in seconds), and gets the same message right back, prefixed +with the zmq identity of the XREQ socket in the heartbeat process. This can be +a uuid, or even a full message, but there doesn't seem to be a need for packing +up a message when the sender and receiver are the exact same Python object. + +The model is this:: + + monitor.send(str(self.lifetime)) # '1.2345678910' + +and the monitor receives some number of messages of the form:: + + ['uuid-abcd-dead-beef', '1.2345678910'] + +where the first part is the zmq.IDENTITY of the heart's XREQ on the engine, and +the rest is the message sent by the monitor. No Python code ever has any +access to the message between the monitor's send, and the monitor's recv. + + +ToDo +==== + +Missing things include: + +* Important: finish thinking through the payload concept and API. + +* Important: ensure that we have a good solution for magics like %edit. It's + likely that with the payload concept we can build a full solution, but not + 100% clear yet. + +* Finishing the details of the heartbeat protocol. + +* Signal handling: specify what kind of information kernel should broadcast (or + not) when it receives signals. + +.. include:: ../links.rst diff --git a/docs/source/links.rst b/docs/source/links.rst new file mode 100644 index 0000000..ce0b5ab --- /dev/null +++ b/docs/source/links.rst @@ -0,0 +1,74 @@ +.. This (-*- rst -*-) format file contains commonly used link targets + and name substitutions. It may be included in many files, + therefore it should only contain link targets and name + substitutions. Try grepping for "^\.\. _" to find plausible + candidates for this list. + + NOTE: this file must have an extension *opposite* to that of the main reST + files in the manuals, so that we can include it with ".. include::" + directives, but without triggering warnings from Sphinx for not being listed + in any toctree. Since IPython uses .txt for the main files, this wone will + use .rst. + + NOTE: reST targets are + __not_case_sensitive__, so only one target definition is needed for + ipython, IPython, etc. + + NOTE: Some of these were taken from the nipy links compendium. + +.. Main IPython links +.. _ipython: http://ipython.scipy.org +.. _`ipython manual`: http://ipython.scipy.org/doc/manual/html +.. _ipython_github: http://github.com/ipython/ipython/ +.. _ipython_github_repo: http://github.com/ipython/ipython/ +.. _ipython_downloads: http://ipython.scipy.org/dist +.. _ipython_pypi: http://pypi.python.org/pypi/ipython + +.. _ZeroMQ: http://zeromq.org + +.. Documentation tools and related links +.. _graphviz: http://www.graphviz.org +.. _Sphinx: http://sphinx.pocoo.org +.. _`Sphinx reST`: http://sphinx.pocoo.org/rest.html +.. _sampledoc: http://matplotlib.sourceforge.net/sampledoc +.. _reST: http://docutils.sourceforge.net/rst.html +.. _docutils: http://docutils.sourceforge.net +.. _lyx: http://www.lyx.org +.. _pep8: http://www.python.org/dev/peps/pep-0008 +.. _numpy_coding_guide: http://projects.scipy.org/numpy/wiki/CodingStyleGuidelines + +.. Licenses +.. _GPL: http://www.gnu.org/licenses/gpl.html +.. _BSD: http://www.opensource.org/licenses/bsd-license.php +.. _LGPL: http://www.gnu.org/copyleft/lesser.html + +.. Other python projects +.. _numpy: http://numpy.scipy.org +.. _scipy: http://www.scipy.org +.. _scipy_conference: http://conference.scipy.org +.. _matplotlib: http://matplotlib.sourceforge.net +.. _pythonxy: http://www.pythonxy.com +.. _ETS: http://code.enthought.com/projects/tool-suite.php +.. _EPD: http://www.enthought.com/products/epd.php +.. _python: http://www.python.org +.. _mayavi: http://code.enthought.com/projects/mayavi +.. _sympy: http://code.google.com/p/sympy +.. _sage: http://sagemath.org +.. _pydy: http://code.google.com/p/pydy +.. _vpython: http://vpython.org +.. _cython: http://cython.org +.. _software carpentry: http://software-carpentry.org + +.. Not so python scientific computing tools +.. _matlab: http://www.mathworks.com +.. _VTK: http://vtk.org + +.. Other organizations +.. _enthought: http://www.enthought.com +.. _kitware: http://www.kitware.com +.. _netlib: http://netlib.org + +.. Other tools and projects +.. _indefero: http://www.indefero.net +.. _git: http://git-scm.com +.. _github: http://github.com diff --git a/docs/sphinxext/ipython_directive.py b/docs/sphinxext/ipython_directive.py index b71b3d3..cc914ee 100644 --- a/docs/sphinxext/ipython_directive.py +++ b/docs/sphinxext/ipython_directive.py @@ -220,7 +220,7 @@ class EmbeddedSphinxShell(object): config.InteractiveShell.colors = 'NoColor' # Create and initialize ipython, but don't start its mainloop - IP = InteractiveShell(parent=None, config=config) + IP = InteractiveShell.instance(config=config) # Store a few parts of IPython we'll need. self.IP = IP