Track upstream

IPython/frontend/cocoa/plugin/CocoaFrontendPlugin.xcodeproj/project.pbxproj

@@ -0,0 +1,256 b''
+// !$*UTF8*$!
+{
+	archiveVersion = 1;
+	classes = {
+	};
+	objectVersion = 42;
+	objects = {
+
+/* Begin PBXContainerItemProxy section */
+		4C5B7ADB0E1A0BCD006CB905 /* PBXContainerItemProxy */ = {
+			isa = PBXContainerItemProxy;
+			containerPortal = 4C96F4FE0E199AB500B03430 /* Project object */;
+			proxyType = 1;
+			remoteGlobalIDString = 4C96F50C0E199AF100B03430;
+			remoteInfo = "Cocoa Frontend Plugin";
+		};
+/* End PBXContainerItemProxy section */
+
+/* Begin PBXFileReference section */
+		4C5B7A8D0E1A0B4C006CB905 /* Plugin-Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = "Plugin-Info.plist"; sourceTree = "<group>"; };
+		4C5B7AD30E1A0BC8006CB905 /* Placeholder (Do Not Use).bundle */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = "Placeholder (Do Not Use).bundle"; sourceTree = BUILT_PRODUCTS_DIR; };
+		4C5B7AD40E1A0BC8006CB905 /* Placeholder (Do Not Use)-Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = "Placeholder (Do Not Use)-Info.plist"; sourceTree = "<group>"; };
+/* End PBXFileReference section */
+
+/* Begin PBXFrameworksBuildPhase section */
+		4C5B7AD10E1A0BC8006CB905 /* Frameworks */ = {
+			isa = PBXFrameworksBuildPhase;
+			buildActionMask = 2147483647;
+			files = (
+			);
+			runOnlyForDeploymentPostprocessing = 0;
+		};
+/* End PBXFrameworksBuildPhase section */
+
+/* Begin PBXGroup section */
+		4C5B7A8C0E1A0B4C006CB905 /* Products */ = {
+			isa = PBXGroup;
+			children = (
+				4C5B7AD30E1A0BC8006CB905 /* Placeholder (Do Not Use).bundle */,
+			);
+			name = Products;
+			sourceTree = "<group>";
+		};
+		4C96F4FC0E199AB500B03430 = {
+			isa = PBXGroup;
+			children = (
+				4C5B7A8C0E1A0B4C006CB905 /* Products */,
+				4C5B7A8D0E1A0B4C006CB905 /* Plugin-Info.plist */,
+				4C5B7AD40E1A0BC8006CB905 /* Placeholder (Do Not Use)-Info.plist */,
+			);
+			sourceTree = "<group>";
+		};
+/* End PBXGroup section */
+
+/* Begin PBXLegacyTarget section */
+		4C96F50C0E199AF100B03430 /* Cocoa Frontend Plugin */ = {
+			isa = PBXLegacyTarget;
+			buildArgumentsString = "$(ACTION)";
+			buildConfigurationList = 4C96F5110E199B3300B03430 /* Build configuration list for PBXLegacyTarget "Cocoa Frontend Plugin" */;
+			buildPhases = (
+			);
+			buildToolPath = /usr/bin/make;
+			buildWorkingDirectory = "";
+			dependencies = (
+			);
+			name = "Cocoa Frontend Plugin";
+			passBuildSettingsInEnvironment = 1;
+			productName = "Cocoa Frontend Plugin";
+		};
+/* End PBXLegacyTarget section */
+
+/* Begin PBXNativeTarget section */
+		4C5B7AD20E1A0BC8006CB905 /* Placeholder (Do Not Use) */ = {
+			isa = PBXNativeTarget;
+			buildConfigurationList = 4C5B7ADA0E1A0BC9006CB905 /* Build configuration list for PBXNativeTarget "Placeholder (Do Not Use)" */;
+			buildPhases = (
+				4C5B7ACF0E1A0BC8006CB905 /* Resources */,
+				4C5B7AD00E1A0BC8006CB905 /* Sources */,
+				4C5B7AD10E1A0BC8006CB905 /* Frameworks */,
+			);
+			buildRules = (
+			);
+			dependencies = (
+				4C5B7ADC0E1A0BCD006CB905 /* PBXTargetDependency */,
+			);
+			name = "Placeholder (Do Not Use)";
+			productName = "Placeholder (Do Not Use)";
+			productReference = 4C5B7AD30E1A0BC8006CB905 /* Placeholder (Do Not Use).bundle */;
+			productType = "com.apple.product-type.bundle";
+		};
+/* End PBXNativeTarget section */
+
+/* Begin PBXProject section */
+		4C96F4FE0E199AB500B03430 /* Project object */ = {
+			isa = PBXProject;
+			buildConfigurationList = 4C96F5010E199AB500B03430 /* Build configuration list for PBXProject "CocoaFrontendPlugin" */;
+			compatibilityVersion = "Xcode 2.4";
+			hasScannedForEncodings = 0;
+			mainGroup = 4C96F4FC0E199AB500B03430;
+			productRefGroup = 4C5B7A8C0E1A0B4C006CB905 /* Products */;
+			projectDirPath = "";
+			projectRoot = "";
+			targets = (
+				4C96F50C0E199AF100B03430 /* Cocoa Frontend Plugin */,
+				4C5B7AD20E1A0BC8006CB905 /* Placeholder (Do Not Use) */,
+			);
+		};
+/* End PBXProject section */
+
+/* Begin PBXResourcesBuildPhase section */
+		4C5B7ACF0E1A0BC8006CB905 /* Resources */ = {
+			isa = PBXResourcesBuildPhase;
+			buildActionMask = 2147483647;
+			files = (
+			);
+			runOnlyForDeploymentPostprocessing = 0;
+		};
+/* End PBXResourcesBuildPhase section */
+
+/* Begin PBXSourcesBuildPhase section */
+		4C5B7AD00E1A0BC8006CB905 /* Sources */ = {
+			isa = PBXSourcesBuildPhase;
+			buildActionMask = 2147483647;
+			files = (
+			);
+			runOnlyForDeploymentPostprocessing = 0;
+		};
+/* End PBXSourcesBuildPhase section */
+
+/* Begin PBXTargetDependency section */
+		4C5B7ADC0E1A0BCD006CB905 /* PBXTargetDependency */ = {
+			isa = PBXTargetDependency;
+			target = 4C96F50C0E199AF100B03430 /* Cocoa Frontend Plugin */;
+			targetProxy = 4C5B7ADB0E1A0BCD006CB905 /* PBXContainerItemProxy */;
+		};
+/* End PBXTargetDependency section */
+
+/* Begin XCBuildConfiguration section */
+		4C5B7AD50E1A0BC9006CB905 /* Debug */ = {
+			isa = XCBuildConfiguration;
+			buildSettings = {
+				COPY_PHASE_STRIP = NO;
+				GCC_DYNAMIC_NO_PIC = NO;
+				GCC_ENABLE_FIX_AND_CONTINUE = YES;
+				GCC_MODEL_TUNING = G5;
+				GCC_OPTIMIZATION_LEVEL = 0;
+				GCC_PRECOMPILE_PREFIX_HEADER = YES;
+				GCC_PREFIX_HEADER = "$(SYSTEM_LIBRARY_DIR)/Frameworks/AppKit.framework/Headers/AppKit.h";
+				INFOPLIST_FILE = "Placeholder (Do Not Use)-Info.plist";
+				INSTALL_PATH = "$(HOME)/Library/Bundles";
+				OTHER_LDFLAGS = (
+					"-framework",
+					Foundation,
+					"-framework",
+					AppKit,
+				);
+				PREBINDING = NO;
+				PRODUCT_NAME = "Placeholder (Do Not Use)";
+				WRAPPER_EXTENSION = bundle;
+				ZERO_LINK = YES;
+			};
+			name = Debug;
+		};
+		4C5B7AD60E1A0BC9006CB905 /* Release */ = {
+			isa = XCBuildConfiguration;
+			buildSettings = {
+				COPY_PHASE_STRIP = YES;
+				DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym";
+				GCC_ENABLE_FIX_AND_CONTINUE = NO;
+				GCC_MODEL_TUNING = G5;
+				GCC_PRECOMPILE_PREFIX_HEADER = YES;
+				GCC_PREFIX_HEADER = "$(SYSTEM_LIBRARY_DIR)/Frameworks/AppKit.framework/Headers/AppKit.h";
+				INFOPLIST_FILE = "Placeholder (Do Not Use)-Info.plist";
+				INSTALL_PATH = "$(HOME)/Library/Bundles";
+				OTHER_LDFLAGS = (
+					"-framework",
+					Foundation,
+					"-framework",
+					AppKit,
+				);
+				PREBINDING = NO;
+				PRODUCT_NAME = "Placeholder (Do Not Use)";
+				WRAPPER_EXTENSION = bundle;
+				ZERO_LINK = NO;
+			};
+			name = Release;
+		};
+		4C96F4FF0E199AB500B03430 /* Debug */ = {
+			isa = XCBuildConfiguration;
+			buildSettings = {
+				COPY_PHASE_STRIP = NO;
+			};
+			name = Debug;
+		};
+		4C96F5000E199AB500B03430 /* Release */ = {
+			isa = XCBuildConfiguration;
+			buildSettings = {
+				COPY_PHASE_STRIP = YES;
+			};
+			name = Release;
+		};
+		4C96F50D0E199AF100B03430 /* Debug */ = {
+			isa = XCBuildConfiguration;
+			buildSettings = {
+				COPY_PHASE_STRIP = NO;
+				GCC_DYNAMIC_NO_PIC = NO;
+				GCC_OPTIMIZATION_LEVEL = 0;
+				PRODUCT_NAME = "Cocoa Frontend Plugin";
+			};
+			name = Debug;
+		};
+		4C96F50E0E199AF100B03430 /* Release */ = {
+			isa = XCBuildConfiguration;
+			buildSettings = {
+				COPY_PHASE_STRIP = YES;
+				DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym";
+				GCC_ENABLE_FIX_AND_CONTINUE = NO;
+				PRODUCT_NAME = "Cocoa Frontend Plugin";
+				ZERO_LINK = NO;
+			};
+			name = Release;
+		};
+/* End XCBuildConfiguration section */
+
+/* Begin XCConfigurationList section */
+		4C5B7ADA0E1A0BC9006CB905 /* Build configuration list for PBXNativeTarget "Placeholder (Do Not Use)" */ = {
+			isa = XCConfigurationList;
+			buildConfigurations = (
+				4C5B7AD50E1A0BC9006CB905 /* Debug */,
+				4C5B7AD60E1A0BC9006CB905 /* Release */,
+			);
+			defaultConfigurationIsVisible = 0;
+			defaultConfigurationName = Release;
+		};
+		4C96F5010E199AB500B03430 /* Build configuration list for PBXProject "CocoaFrontendPlugin" */ = {
+			isa = XCConfigurationList;
+			buildConfigurations = (
+				4C96F4FF0E199AB500B03430 /* Debug */,
+				4C96F5000E199AB500B03430 /* Release */,
+			);
+			defaultConfigurationIsVisible = 0;
+			defaultConfigurationName = Release;
+		};
+		4C96F5110E199B3300B03430 /* Build configuration list for PBXLegacyTarget "Cocoa Frontend Plugin" */ = {
+			isa = XCConfigurationList;
+			buildConfigurations = (
+				4C96F50D0E199AF100B03430 /* Debug */,
+				4C96F50E0E199AF100B03430 /* Release */,
+			);
+			defaultConfigurationIsVisible = 0;
+			defaultConfigurationName = Release;
+		};
+/* End XCConfigurationList section */
+	};
+	rootObject = 4C96F4FE0E199AB500B03430 /* Project object */;
+}

IPython/frontend/cocoa/plugin/IPythonCocoaFrontendLoader.py

@@ -0,0 +1,25 b''
+# encoding: utf-8
+"""
+Provides a namespace for loading the Cocoa frontend via a Cocoa plugin.
+
+Author: Barry Wark
+"""
+__docformat__ = "restructuredtext en"
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+from PyObjCTools import AppHelper
+from twisted.internet import _threadedselect
+
+#make sure _threadedselect is installed first
+reactor = _threadedselect.install()
+
+# load the Cocoa frontend controller
+from IPython.frontend.cocoa.cocoa_frontend import IPythonCocoaController
+reactor.interleave(AppHelper.callAfter)
+assert(reactor.running)

IPython/frontend/cocoa/plugin/Makefile

@@ -0,0 +1,6 b''
+include ./plugins.mk
+
+all : dist/IPythonCocoaController.plugin
+
+dist/IPythonCocoaController.plugin : ./IPythonCocoaFrontendLoader.py\
+ ./setup.py
  No newline at end of file

IPython/frontend/cocoa/plugin/Placeholder (Do Not Use)-Info.plist

@@ -0,0 +1,20 b''
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>CFBundleDevelopmentRegion</key>
+	<string>English</string>
+	<key>CFBundleExecutable</key>
+	<string>${EXECUTABLE_NAME}</string>
+	<key>CFBundleIdentifier</key>
+	<string>com.yourcompany.Placeholder (Do Not Use)</string>
+	<key>CFBundleInfoDictionaryVersion</key>
+	<string>6.0</string>
+	<key>CFBundlePackageType</key>
+	<string>BNDL</string>
+	<key>CFBundleSignature</key>
+	<string>????</string>
+	<key>CFBundleVersion</key>
+	<string>1.0</string>
+</dict>
+</plist>

IPython/frontend/cocoa/plugin/plugins.mk

@@ -0,0 +1,21 b''
+%.plugin::
+	rm -rf dist/$(notdir $@)
+	rm -rf build dist && \
+	python setup.py py2app -s
+
+%.py:
+	@echo "test -f $@"
+	@test -f %@
+
+%.nib:
+	@echo "test -f $@"
+	@test -f %@
+
+.DEFAULT_GOAL := all
+
+.PHONY : all clean
+
+clean :
+	rm -rf build dist
+
+

IPython/frontend/cocoa/plugin/setup.py

@@ -0,0 +1,35 b''
+# encoding: utf-8
+"""
+setup.py
+
+Setuptools installer script for generating a Cocoa plugin for the 
+IPython cocoa frontend
+
+Author: Barry Wark
+"""
+__docformat__ = "restructuredtext en"
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+from setuptools import setup
+
+infoPlist = dict(
+	CFBundleDevelopmentRegion='English',
+	CFBundleIdentifier='org.scipy.ipython.cocoa_frontend',
+	NSPrincipalClass='IPythonCocoaController',
+)
+
+setup(
+	plugin=['IPythonCocoaFrontendLoader.py'],
+    setup_requires=['py2app'],
+	options=dict(py2app=dict(
+		plist=infoPlist,
+		site_packages=True,
+		excludes=['IPython','twisted','PyObjCTools']
+	)),
+)
  No newline at end of file

IPython/testing/plugin/Makefile

@@ -0,0 +1,20 b''
+# Set this prefix to where you want to install the plugin
+PREFIX=~/usr/local
+PREFIX=~/tmp/local
+
+plugin: IPython_doctest_plugin.egg-info
+
+test: plugin dtexample.py
+	nosetests -s --with-ipdoctest --doctest-tests --doctest-extension=txt \
+	dtexample.py test*.txt
+
+deb: plugin dtexample.py
+	nosetests -vs --with-ipdoctest --doctest-tests --doctest-extension=txt \
+        test_combo.txt
+
+IPython_doctest_plugin.egg-info: ipdoctest.py setup.py
+	python setup.py install --prefix=$(PREFIX)
+	touch $@
+
+clean:
+	rm -rf IPython_doctest_plugin.egg-info *~ *pyc build/ dist/

IPython/testing/plugin/README.txt

@@ -0,0 +1,39 b''
+=======================================================
+ Nose plugin with IPython and extension module support
+=======================================================
+
+This directory provides the key functionality for test support that IPython
+needs as a nose plugin, which can be installed for use in projects other than
+IPython.
+
+The presence of a Makefile here is mostly for development and debugging
+purposes as it only provides a few shorthand commands.  You can manually
+install the plugin by using standard Python procedures (``setup.py install``
+with appropriate arguments).
+
+To install the plugin using the Makefile, edit its first line to reflect where
+you'd like the installation.  If you want it system-wide, you may want to edit
+the install line in the plugin target to use sudo and no prefix::
+
+  sudo python setup.py install
+
+instead of the code using `--prefix` that's in there.
+
+Once you've set the prefix, simply build/install the plugin with::
+
+  make
+
+and run the tests with::
+
+  make test
+
+You should see output similar to::
+
+  maqroll[plugin]> make test
+  nosetests -s --with-ipdoctest --doctest-tests dtexample.py
+  ..
+  ----------------------------------------------------------------------
+  Ran 2 tests in 0.016s
+
+  OK
+

IPython/testing/plugin/dtexample.py

@@ -0,0 +1,72 b''
+"""Simple example using doctests.
+
+This file just contains doctests both using plain python and IPython prompts.
+All tests should be loaded by nose.
+"""
+
+def pyfunc():
+    """Some pure python tests...
+
+    >>> pyfunc()
+    'pyfunc'
+
+    >>> import os
+
+    >>> 2+3
+    5
+
+    >>> for i in range(3):
+    ...     print i,
+    ...     print i+1,
+    ...
+    0 1 1 2 2 3
+    """
+
+    return 'pyfunc'
+
+def ipfunc():
+    """Some ipython tests...
+
+    In [1]: import os
+
+    In [2]: cd /
+    /
+
+    In [3]: 2+3
+    Out[3]: 5
+
+    In [26]: for i in range(3):
+       ....:     print i,
+       ....:     print i+1,
+       ....:
+    0 1 1 2 2 3
+
+
+    Examples that access the operating system work:
+
+    In [1]: !echo hello
+    hello
+
+    In [2]: !echo hello > /tmp/foo
+
+    In [3]: !cat /tmp/foo
+    hello
+
+    In [4]: rm -f /tmp/foo
+
+    It's OK to use '_' for the last result, but do NOT try to use IPython's
+    numbered history of _NN outputs, since those won't exist under the
+    doctest environment:
+
+    In [7]: 3+4
+    Out[7]: 7
+
+    In [8]: _+3
+    Out[8]: 10
+
+    In [9]: ipfunc()
+    Out[9]: 'ipfunc'
+    """
+
+    return 'ipfunc'
+

IPython/testing/plugin/ipdoctest.py

@@ -0,0 +1,587 b''
+"""Nose Plugin that supports IPython doctests.
+
+Limitations:
+
+- When generating examples for use as doctests, make sure that you have
+  pretty-printing OFF.  This can be done either by starting ipython with the
+  flag '--nopprint', by setting pprint to 0 in your ipythonrc file, or by
+  interactively disabling it with %Pprint.  This is required so that IPython
+  output matches that of normal Python, which is used by doctest for internal
+  execution.
+
+- Do not rely on specific prompt numbers for results (such as using
+  '_34==True', for example).  For IPython tests run via an external process the
+  prompt numbers may be different, and IPython tests run as normal python code
+  won't even have these special _NN variables set at all.
+
+- IPython functions that produce output as a side-effect of calling a system
+  process (e.g. 'ls') can be doc-tested, but they must be handled in an
+  external IPython process.  Such doctests must be tagged with:
+
+        # ipdoctest: EXTERNAL
+
+  so that the testing machinery handles them differently.  Since these are run
+  via pexpect in an external process, they can't deal with exceptions or other
+  fancy featurs of regular doctests.  You must limit such tests to simple
+  matching of the output.  For this reason, I recommend you limit these kinds
+  of doctests to features that truly require a separate process, and use the
+  normal IPython ones (which have all the features of normal doctests) for
+  everything else.  See the examples at the bottom of this file for a
+  comparison of what can be done with both types.
+"""
+
+
+#-----------------------------------------------------------------------------
+# Module imports
+
+# From the standard library
+import __builtin__
+import commands
+import doctest
+import inspect
+import logging
+import os
+import re
+import sys
+import unittest
+
+from inspect import getmodule
+
+# Third-party modules
+import nose.core
+
+from nose.plugins import doctests, Plugin
+from nose.util import anyp, getpackage, test_address, resolve_name, tolist
+
+# Our own imports
+#from extdoctest import ExtensionDoctest, DocTestFinder
+#from dttools import DocTestFinder, DocTestCase
+#-----------------------------------------------------------------------------
+# Module globals and other constants
+
+log = logging.getLogger(__name__)
+
+###########################################################################
+# *** HACK ***
+# We must start our own ipython object and heavily muck with it so that all the
+# modifications IPython makes to system behavior don't send the doctest
+# machinery into a fit.  This code should be considered a gross hack, but it
+# gets the job done.
+
+def start_ipython():
+    """Start a global IPython shell, which we need for IPython-specific syntax.
+    """
+    import IPython
+
+    def xsys(cmd):
+        """Execute a command and print its output.
+
+        This is just a convenience function to replace the IPython system call
+        with one that is more doctest-friendly.
+        """
+        cmd = _ip.IP.var_expand(cmd,depth=1)
+        sys.stdout.write(commands.getoutput(cmd))
+        sys.stdout.flush()
+
+    # Store certain global objects that IPython modifies
+    _displayhook = sys.displayhook
+    _excepthook = sys.excepthook
+    _main = sys.modules.get('__main__')
+
+    # Start IPython instance
+    IPython.Shell.IPShell(['--classic','--noterm_title'])
+
+    # Deactivate the various python system hooks added by ipython for
+    # interactive convenience so we don't confuse the doctest system
+    sys.modules['__main__'] = _main
+    sys.displayhook = _displayhook
+    sys.excepthook = _excepthook
+
+    # So that ipython magics and aliases can be doctested (they work by making
+    # a call into a global _ip object)
+    _ip = IPython.ipapi.get()
+    __builtin__._ip = _ip
+
+    # Modify the IPython system call with one that uses getoutput, so that we
+    # can capture subcommands and print them to Python's stdout, otherwise the
+    # doctest machinery would miss them.
+    _ip.system = xsys
+
+# The start call MUST be made here.  I'm not sure yet why it doesn't work if
+# it is made later, at plugin initialization time, but in all my tests, that's
+# the case.
+start_ipython()
+
+# *** END HACK ***
+###########################################################################
+
+#-----------------------------------------------------------------------------
+# Modified version of the one in the stdlib, that fixes a python bug (doctests
+# not found in extension modules, http://bugs.python.org/issue3158)
+class DocTestFinder(doctest.DocTestFinder):
+
+    def _from_module(self, module, object):
+        """
+        Return true if the given object is defined in the given
+        module.
+        """
+        if module is None:
+            #print '_fm C1'  # dbg
+            return True
+        elif inspect.isfunction(object):
+            #print '_fm C2'  # dbg
+            return module.__dict__ is object.func_globals
+        elif inspect.isbuiltin(object):
+            #print '_fm C2-1'  # dbg
+            return module.__name__ == object.__module__
+        elif inspect.isclass(object):
+            #print '_fm C3'  # dbg
+            return module.__name__ == object.__module__
+        elif inspect.ismethod(object):
+            # This one may be a bug in cython that fails to correctly set the
+            # __module__ attribute of methods, but since the same error is easy
+            # to make by extension code writers, having this safety in place
+            # isn't such a bad idea
+            #print '_fm C3-1'  # dbg
+            return module.__name__ == object.im_class.__module__
+        elif inspect.getmodule(object) is not None:
+            #print '_fm C4'  # dbg
+            #print 'C4 mod',module,'obj',object # dbg
+            return module is inspect.getmodule(object)
+        elif hasattr(object, '__module__'):
+            #print '_fm C5'  # dbg
+            return module.__name__ == object.__module__
+        elif isinstance(object, property):
+            #print '_fm C6'  # dbg
+            return True # [XX] no way not be sure.
+        else:
+            raise ValueError("object must be a class or function")
+
+
+
+    def _find(self, tests, obj, name, module, source_lines, globs, seen):
+        """
+        Find tests for the given object and any contained objects, and
+        add them to `tests`.
+        """
+
+        doctest.DocTestFinder._find(self,tests, obj, name, module,
+                                    source_lines, globs, seen)
+
+        # Below we re-run pieces of the above method with manual modifications,
+        # because the original code is buggy and fails to correctly identify
+        # doctests in extension modules.
+
+        # Local shorthands
+        from inspect import isroutine, isclass, ismodule
+
+        # Look for tests in a module's contained objects.
+        if inspect.ismodule(obj) and self._recurse:
+            for valname, val in obj.__dict__.items():
+                valname1 = '%s.%s' % (name, valname)
+                if ( (isroutine(val) or isclass(val))
+                     and self._from_module(module, val) ):
+
+                    self._find(tests, val, valname1, module, source_lines,
+                               globs, seen)
+
+
+        # Look for tests in a class's contained objects.
+        if inspect.isclass(obj) and self._recurse:
+            #print 'RECURSE into class:',obj  # dbg
+            for valname, val in obj.__dict__.items():
+                #valname1 = '%s.%s' % (name, valname)  # dbg
+                #print 'N',name,'VN:',valname,'val:',str(val)[:77] # dbg
+                # Special handling for staticmethod/classmethod.
+                if isinstance(val, staticmethod):
+                    val = getattr(obj, valname)
+                if isinstance(val, classmethod):
+                    val = getattr(obj, valname).im_func
+
+                # Recurse to methods, properties, and nested classes.
+                if ((inspect.isfunction(val) or inspect.isclass(val) or
+                     inspect.ismethod(val) or
+                      isinstance(val, property)) and
+                      self._from_module(module, val)):
+                    valname = '%s.%s' % (name, valname)
+                    self._find(tests, val, valname, module, source_lines,
+                               globs, seen)
+
+
+class DocTestCase(doctests.DocTestCase):
+    """Proxy for DocTestCase: provides an address() method that
+    returns the correct address for the doctest case. Otherwise
+    acts as a proxy to the test case. To provide hints for address(),
+    an obj may also be passed -- this will be used as the test object
+    for purposes of determining the test address, if it is provided.
+    """
+
+    # doctests loaded via find(obj) omit the module name
+    # so we need to override id, __repr__ and shortDescription
+    # bonus: this will squash a 2.3 vs 2.4 incompatiblity
+    def id(self):
+        name = self._dt_test.name
+        filename = self._dt_test.filename
+        if filename is not None:
+            pk = getpackage(filename)
+            if pk is not None and not name.startswith(pk):
+                name = "%s.%s" % (pk, name)
+        return name
+
+
+# Classes and functions
+
+def is_extension_module(filename):
+    """Return whether the given filename is an extension module.
+
+    This simply checks that the extension is either .so or .pyd.
+    """
+    return os.path.splitext(filename)[1].lower() in ('.so','.pyd')
+
+
+# A simple subclassing of the original with a different class name, so we can
+# distinguish and treat differently IPython examples from pure python ones.
+class IPExample(doctest.Example): pass
+
+class IPExternalExample(doctest.Example):
+    """Doctest examples to be run in an external process."""
+
+    def __init__(self, source, want, exc_msg=None, lineno=0, indent=0,
+                 options=None):
+        # Parent constructor
+        doctest.Example.__init__(self,source,want,exc_msg,lineno,indent,options)
+
+        # An EXTRA newline is needed to prevent pexpect hangs
+        self.source += '\n'
+
+class IPDocTestParser(doctest.DocTestParser):
+    """
+    A class used to parse strings containing doctest examples.
+
+    Note: This is a version modified to properly recognize IPython input and
+    convert any IPython examples into valid Python ones.
+    """
+    # This regular expression is used to find doctest examples in a
+    # string.  It defines three groups: `source` is the source code
+    # (including leading indentation and prompts); `indent` is the
+    # indentation of the first (PS1) line of the source code; and
+    # `want` is the expected output (including leading indentation).
+
+    # Classic Python prompts or default IPython ones
+    _PS1_PY = r'>>>'
+    _PS2_PY = r'\.\.\.'
+
+    _PS1_IP = r'In\ \[\d+\]:'
+    _PS2_IP = r'\ \ \ \.\.\.+:'
+
+    _RE_TPL = r'''
+        # Source consists of a PS1 line followed by zero or more PS2 lines.
+        (?P<source>
+            (?:^(?P<indent> [ ]*) (?P<ps1> %s) .*)    # PS1 line
+            (?:\n           [ ]*  (?P<ps2> %s) .*)*)  # PS2 lines
+        \n? # a newline
+        # Want consists of any non-blank lines that do not start with PS1.
+        (?P<want> (?:(?![ ]*$)    # Not a blank line
+                     (?![ ]*%s)   # Not a line starting with PS1
+                     (?![ ]*%s)   # Not a line starting with PS2
+                     .*$\n?       # But any other line
+                  )*)
+                  '''
+
+    _EXAMPLE_RE_PY = re.compile( _RE_TPL % (_PS1_PY,_PS2_PY,_PS1_PY,_PS2_PY),
+                                 re.MULTILINE | re.VERBOSE)
+
+    _EXAMPLE_RE_IP = re.compile( _RE_TPL % (_PS1_IP,_PS2_IP,_PS1_IP,_PS2_IP),
+                                 re.MULTILINE | re.VERBOSE)
+
+    def ip2py(self,source):
+        """Convert input IPython source into valid Python."""
+        out = []
+        newline = out.append
+        for line in source.splitlines():
+            #newline(_ip.IPipython.prefilter(line,True))
+            newline(_ip.IP.prefilter(line,True))
+        newline('')  # ensure a closing newline, needed by doctest
+        return '\n'.join(out)
+
+    def parse(self, string, name='<string>'):
+        """
+        Divide the given string into examples and intervening text,
+        and return them as a list of alternating Examples and strings.
+        Line numbers for the Examples are 0-based.  The optional
+        argument `name` is a name identifying this string, and is only
+        used for error messages.
+        """
+
+        #print 'Parse string:\n',string # dbg
+
+        string = string.expandtabs()
+        # If all lines begin with the same indentation, then strip it.
+        min_indent = self._min_indent(string)
+        if min_indent > 0:
+            string = '\n'.join([l[min_indent:] for l in string.split('\n')])
+
+        output = []
+        charno, lineno = 0, 0
+
+        # Whether to convert the input from ipython to python syntax
+        ip2py = False
+        # Find all doctest examples in the string.  First, try them as Python
+        # examples, then as IPython ones
+        terms = list(self._EXAMPLE_RE_PY.finditer(string))
+        if terms:
+            # Normal Python example
+            #print '-'*70  # dbg
+            #print 'PyExample, Source:\n',string  # dbg
+            #print '-'*70  # dbg
+            Example = doctest.Example
+        else:
+            # It's an ipython example.  Note that IPExamples are run
+            # in-process, so their syntax must be turned into valid python.
+            # IPExternalExamples are run out-of-process (via pexpect) so they
+            # don't need any filtering (a real ipython will be executing them).
+            terms = list(self._EXAMPLE_RE_IP.finditer(string))
+            if re.search(r'#\s*ipdoctest:\s*EXTERNAL',string):
+                #print '-'*70  # dbg
+                #print 'IPExternalExample, Source:\n',string  # dbg
+                #print '-'*70  # dbg
+                Example = IPExternalExample
+            else:
+                #print '-'*70  # dbg
+                #print 'IPExample, Source:\n',string  # dbg
+                #print '-'*70  # dbg
+                Example = IPExample
+                ip2py = True
+
+        for m in terms:
+            # Add the pre-example text to `output`.
+            output.append(string[charno:m.start()])
+            # Update lineno (lines before this example)
+            lineno += string.count('\n', charno, m.start())
+            # Extract info from the regexp match.
+            (source, options, want, exc_msg) = \
+                     self._parse_example(m, name, lineno,ip2py)
+            if Example is IPExternalExample:
+                options[doctest.NORMALIZE_WHITESPACE] = True
+                want += '\n'
+            # Create an Example, and add it to the list.
+            if not self._IS_BLANK_OR_COMMENT(source):
+                #print 'Example source:', source # dbg
+                output.append(Example(source, want, exc_msg,
+                                      lineno=lineno,
+                                      indent=min_indent+len(m.group('indent')),
+                                      options=options))
+            # Update lineno (lines inside this example)
+            lineno += string.count('\n', m.start(), m.end())
+            # Update charno.
+            charno = m.end()
+        # Add any remaining post-example text to `output`.
+        output.append(string[charno:])
+
+        return output
+
+    def _parse_example(self, m, name, lineno,ip2py=False):
+        """
+        Given a regular expression match from `_EXAMPLE_RE` (`m`),
+        return a pair `(source, want)`, where `source` is the matched
+        example's source code (with prompts and indentation stripped);
+        and `want` is the example's expected output (with indentation
+        stripped).
+
+        `name` is the string's name, and `lineno` is the line number
+        where the example starts; both are used for error messages.
+
+        Optional:
+        `ip2py`: if true, filter the input via IPython to convert the syntax
+        into valid python.
+        """
+
+        # Get the example's indentation level.
+        indent = len(m.group('indent'))
+
+        # Divide source into lines; check that they're properly
+        # indented; and then strip their indentation & prompts.
+        source_lines = m.group('source').split('\n')
+
+        # We're using variable-length input prompts
+        ps1 = m.group('ps1')
+        ps2 = m.group('ps2')
+        ps1_len = len(ps1)
+
+        self._check_prompt_blank(source_lines, indent, name, lineno,ps1_len)
+        if ps2:
+            self._check_prefix(source_lines[1:], ' '*indent + ps2, name, lineno)
+
+        source = '\n'.join([sl[indent+ps1_len+1:] for sl in source_lines])
+
+        if ip2py:
+            # Convert source input from IPython into valid Python syntax
+            source = self.ip2py(source)
+
+        # Divide want into lines; check that it's properly indented; and
+        # then strip the indentation.  Spaces before the last newline should
+        # be preserved, so plain rstrip() isn't good enough.
+        want = m.group('want')
+        want_lines = want.split('\n')
+        if len(want_lines) > 1 and re.match(r' *$', want_lines[-1]):
+            del want_lines[-1]  # forget final newline & spaces after it
+        self._check_prefix(want_lines, ' '*indent, name,
+                           lineno + len(source_lines))
+
+        # Remove ipython output prompt that might be present in the first line
+        want_lines[0] = re.sub(r'Out\[\d+\]: \s*?\n?','',want_lines[0])
+
+        want = '\n'.join([wl[indent:] for wl in want_lines])
+
+        # If `want` contains a traceback message, then extract it.
+        m = self._EXCEPTION_RE.match(want)
+        if m:
+            exc_msg = m.group('msg')
+        else:
+            exc_msg = None
+
+        # Extract options from the source.
+        options = self._find_options(source, name, lineno)
+
+        return source, options, want, exc_msg
+
+    def _check_prompt_blank(self, lines, indent, name, lineno, ps1_len):
+        """
+        Given the lines of a source string (including prompts and
+        leading indentation), check to make sure that every prompt is
+        followed by a space character.  If any line is not followed by
+        a space character, then raise ValueError.
+
+        Note: IPython-modified version which takes the input prompt length as a
+        parameter, so that prompts of variable length can be dealt with.
+        """
+        space_idx = indent+ps1_len
+        min_len = space_idx+1
+        for i, line in enumerate(lines):
+            if len(line) >=  min_len and line[space_idx] != ' ':
+                raise ValueError('line %r of the docstring for %s '
+                                 'lacks blank after %s: %r' %
+                                 (lineno+i+1, name,
+                                  line[indent:space_idx], line))
+
+SKIP = doctest.register_optionflag('SKIP')
+
+###########################################################################
+
+class DocFileCase(doctest.DocFileCase):
+    """Overrides to provide filename
+    """
+    def address(self):
+        return (self._dt_test.filename, None, None)
+
+
+class ExtensionDoctest(doctests.Doctest):
+    """Nose Plugin that supports doctests in extension modules.
+    """
+    name = 'extdoctest'   # call nosetests with --with-extdoctest
+    enabled = True
+
+    def options(self, parser, env=os.environ):
+        Plugin.options(self, parser, env)
+
+    def configure(self, options, config):
+        Plugin.configure(self, options, config)
+        self.doctest_tests = options.doctest_tests
+        self.extension = tolist(options.doctestExtension)
+        self.finder = DocTestFinder()
+        self.parser = doctest.DocTestParser()
+
+
+    def loadTestsFromExtensionModule(self,filename):
+        bpath,mod = os.path.split(filename)
+        modname = os.path.splitext(mod)[0]
+        try:
+            sys.path.append(bpath)
+            module = __import__(modname)
+            tests = list(self.loadTestsFromModule(module))
+        finally:
+            sys.path.pop()
+        return tests
+
+    def loadTestsFromFile(self, filename):
+        if is_extension_module(filename):
+            for t in self.loadTestsFromExtensionModule(filename):
+                yield t
+        else:
+            ## for t in list(doctests.Doctest.loadTestsFromFile(self,filename)):
+            ##     yield t
+            pass
+
+        if self.extension and anyp(filename.endswith, self.extension):
+            #print 'lTF',filename  # dbg
+            name = os.path.basename(filename)
+            dh = open(filename)
+            try:
+                doc = dh.read()
+            finally:
+                dh.close()
+            test = self.parser.get_doctest(
+                doc, globs={'__file__': filename}, name=name,
+                filename=filename, lineno=0)
+            if test.examples:
+                #print 'FileCase:',test.examples  # dbg
+                yield DocFileCase(test)
+            else:
+                yield False # no tests to load
+
+
+    def wantFile(self,filename):
+        """Return whether the given filename should be scanned for tests.
+
+        Modified version that accepts extension modules as valid containers for
+        doctests.
+        """
+        #print 'Filename:',filename  # dbg
+
+        if is_extension_module(filename):
+            return True
+        else:
+            return doctests.Doctest.wantFile(self,filename)
+
+    # NOTE: the method below is a *copy* of the one in the nose doctests
+    # plugin, but we have to replicate it here in order to have it resolve the
+    # DocTestCase (last line) to our local copy, since the nose plugin doesn't
+    # provide a public hook for what TestCase class to use.  The alternative
+    # would be to monkeypatch doctest in the stdlib, but that's ugly and
+    # brittle, since a change in plugin load order can break it.  So for now,
+    # we just paste this in here, inelegant as this may be.
+
+    def loadTestsFromModule(self, module):
+        #print 'lTM',module  # dbg
+
+        if not self.matches(module.__name__):
+            log.debug("Doctest doesn't want module %s", module)
+            return
+        tests = self.finder.find(module)
+        if not tests:
+            return
+        tests.sort()
+        module_file = module.__file__
+        if module_file[-4:] in ('.pyc', '.pyo'):
+            module_file = module_file[:-1]
+        for test in tests:
+            if not test.examples:
+                continue
+            if not test.filename:
+                test.filename = module_file
+            yield DocTestCase(test)
+
+class IPythonDoctest(ExtensionDoctest):
+    """Nose Plugin that supports doctests in extension modules.
+    """
+    name = 'ipdoctest'   # call nosetests with --with-ipdoctest
+    enabled = True
+
+    def configure(self, options, config):
+
+        Plugin.configure(self, options, config)
+        self.doctest_tests = options.doctest_tests
+        self.extension = tolist(options.doctestExtension)
+        self.parser = IPDocTestParser()
+        #self.finder = DocTestFinder(parser=IPDocTestParser())
+        self.finder = DocTestFinder(parser=self.parser)

IPython/testing/plugin/iptest.py

@@ -0,0 +1,18 b''
+#!/usr/bin/env python
+"""Nose-based test runner.
+"""
+
+from nose.core import main
+from nose.plugins.builtin import plugins
+from nose.plugins.doctests import Doctest
+
+import ipdoctest
+from ipdoctest import IPDocTestRunner
+
+if __name__ == '__main__':
+    print 'WARNING: this code is incomplete!'
+    print
+
+    pp = [x() for x in plugins]  # activate all builtin plugins first
+    main(testRunner=IPDocTestRunner(),
+         plugins=pp+[ipdoctest.IPythonDoctest(),Doctest()])

IPython/testing/plugin/setup.py

@@ -0,0 +1,18 b''
+#!/usr/bin/env python
+"""A Nose plugin to support IPython doctests.
+"""
+
+from setuptools import setup
+
+setup(name='IPython doctest plugin',
+      version='0.1',
+      author='The IPython Team',
+      description = 'Nose plugin to load IPython-extended doctests',
+      license = 'LGPL',
+      py_modules = ['ipdoctest'],
+      entry_points = {
+        'nose.plugins.0.10': ['ipdoctest = ipdoctest:IPythonDoctest',
+                              'extdoctest = ipdoctest:ExtensionDoctest',
+                              ],
+        },
+      )

IPython/testing/plugin/test_combo.txt

@@ -0,0 +1,36 b''
+=======================
+ Combo testing example
+=======================
+
+This is a simple example that mixes ipython doctests::
+
+    In [1]: import code
+
+    In [2]: 2**12
+    Out[2]: 4096
+
+with command-line example information that does *not* get executed::
+
+    $ mpirun -n 4 ipengine --controller-port=10000 --controller-ip=host0
+
+and with literal examples of Python source code::
+
+    controller = dict(host='myhost',
+		      engine_port=None, # default is 10105
+		      control_port=None,
+		      )
+
+    # keys are hostnames, values are the number of engine on that host
+    engines = dict(node1=2,
+		   node2=2,
+		   node3=2,
+		   node3=2,
+		   )
+
+    # Force failure to detect that this test is being run.
+    1/0
+
+These source code examples are executed but no output is compared at all.  An
+error or failure is reported only if an exception is raised.
+
+NOTE: the execution of pure python blocks is not yet working!

IPython/testing/plugin/test_example.txt

@@ -0,0 +1,24 b''
+=====================================
+ Tests in example form - pure python
+=====================================
+
+This file contains doctest examples embedded as code blocks, using normal
+Python prompts.  See the accompanying file for similar examples using IPython
+prompts (you can't mix both types within one file). The following will be run
+as a test::
+
+    >>> 1+1
+    2
+    >>> print "hello"
+    hello
+
+More than one example works::
+
+    >>> s="Hello World"
+
+    >>> s.upper()
+    'HELLO WORLD'
+
+but you should note that the *entire* test file is considered to be a single
+test.  Individual code blocks that fail are printed separately as ``example
+failures``, but the whole file is still counted and reported as one test.

IPython/testing/plugin/test_exampleip.txt

@@ -0,0 +1,30 b''
+=================================
+ Tests in example form - IPython
+=================================
+
+You can write text files with examples that use IPython prompts (as long as you
+use the nose ipython doctest plugin), but you can not mix and match prompt
+styles in a single file.  That is, you either use all ``>>>`` prompts or all
+IPython-style prompts.  Your test suite *can* have both types, you just need to
+put each type of example in a separate. Using IPython prompts, you can paste
+directly from your session::
+
+    In [5]: s="Hello World"
+
+    In [6]: s.upper()
+    Out[6]: 'HELLO WORLD'
+
+Another example::
+
+    In [8]: 1+3
+    Out[8]: 4
+
+Just like in IPython docstrings, you can use all IPython syntax and features::
+
+    In [9]: !echo "hello"
+    hello
+
+    In [10]: a='hi'
+
+    In [11]: !echo $a
+    hi

docs/examples/kernel/asyncmultiengine1.py

@@ -0,0 +1,23 b''
+#!/usr/bin/env python
+# encoding: utf-8
+
+# A super simple example showing how to use all of this in a fully
+# asynchronous manner.  The TaskClient also works in this mode.
+
+from twisted.internet import reactor, defer
+from IPython.kernel import asyncclient
+
+def printer(r):
+    print r
+    return r
+
+def submit(client):
+    d = client.push(dict(a=5, b='asdf', c=[1,2,3]),targets=0,block=True)
+    d.addCallback(lambda _: client.pull(('a','b','c'),targets=0,block=True))
+    d.addBoth(printer)
+    d.addCallback(lambda _: reactor.stop())
+
+d = asyncclient.get_multiengine_client()
+d.addCallback(submit)
+
+reactor.run()
  No newline at end of file

docs/examples/kernel/asynctask1.py

@@ -0,0 +1,32 b''
+#!/usr/bin/env python
+# encoding: utf-8
+
+# This example shows how the AsynTaskClient can be used
+# This example is currently broken
+
+from twisted.internet import reactor, defer
+from IPython.kernel import asyncclient
+
+mec = asyncclient.AsyncMultiEngineClient(('localhost', 10105))
+tc = asyncclient.AsyncTaskClient(('localhost',10113))
+
+cmd1 = """\
+a = 5
+b = 10*d
+c = a*b*d
+"""
+
+t1 = asyncclient.Task(cmd1, clear_before=False, clear_after=True, pull=['a','b','c'])
+
+d = mec.push(dict(d=30))
+
+def raise_and_print(tr):
+    tr.raiseException()
+    print "a, b: ", tr.ns.a, tr.ns.b
+    return tr
+
+d.addCallback(lambda _: tc.run(t1))
+d.addCallback(lambda tid: tc.get_task_result(tid,block=True))
+d.addCallback(raise_and_print)
+d.addCallback(lambda _: reactor.stop())
+reactor.run()

IPython/Release.py

@@ -1,95 +1,99 b''
 # -*- coding: utf-8 -*-
 """Release data for the IPython project.
 
 $Id: Release.py 3002 2008-02-01 07:17:00Z fperez $"""
 
 #*****************************************************************************
 #       Copyright (C) 2001-2006 Fernando Perez <[e-mail]>
 #
 #       Copyright (c) 2001 Janko Hauser <[e-mail]> and Nathaniel Gray
 #       <[e-mail]>
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #*****************************************************************************
 
 # Name of the package for release purposes.  This is the name which labels
 # the tarballs and RPMs made by distutils, so it's best to lowercase it.
 name = 'ipython'
 
 # For versions with substrings (like 0.6.16.svn), use an extra . to separate
 # the new substring.  We have to avoid using either dashes or underscores,
 # because bdist_rpm does not accept dashes (an RPM) convention, and
 # bdist_deb does not accept underscores (a Debian convention).
 
-revision = '1016'
+development = True    # change this to False to do a release
+version_base = '0.9.0'
 branch = 'ipython'
+revision = '1016'
 
-if branch == 'ipython':
-    version = '0.9.0.bzr.r' + revision
+if development:
+    if branch == 'ipython':
+        version = '%s.bzr.r%s' % (version_base, revision)
+    else:
+        version = '%s.bzr.r%s.%s' % (version_base, revision, branch)
 else:
-    version = '0.9.0.bzr.r%s.%s'  % (revision,branch)
+    version = version_base
 
-# version = '0.8.4'
 
 description = "Tools for interactive development in Python."
 
 long_description = \
 """
 IPython provides a replacement for the interactive Python interpreter with
 extra functionality.
 
 Main features:
 
  * Comprehensive object introspection.
 
  * Input history, persistent across sessions.
 
  * Caching of output results during a session with automatically generated
    references.
 
  * Readline based name completion.
 
  * Extensible system of 'magic' commands for controlling the environment and
    performing many tasks related either to IPython or the operating system.
 
  * Configuration system with easy switching between different setups (simpler
    than changing $PYTHONSTARTUP environment variables every time).
 
  * Session logging and reloading.
 
  * Extensible syntax processing for special purpose situations.
 
  * Access to the system shell with user-extensible alias system.
 
  * Easily embeddable in other Python programs.
 
  * Integrated access to the pdb debugger and the Python profiler. 
 
  The latest development version is always available at the IPython subversion
  repository_.
 
 .. _repository: http://ipython.scipy.org/svn/ipython/ipython/trunk#egg=ipython-dev
  """
 
 license = 'BSD'
 
 authors = {'Fernando' : ('Fernando Perez','[e-mail]'),
            'Janko'    : ('Janko Hauser','[e-mail]'),
            'Nathan'   : ('Nathaniel Gray','[e-mail]'),
            'Ville'    : ('Ville Vainio','[e-mail]'),
            'Brian'    : ('Brian E Granger', '[e-mail]'),
            'Min'      : ('Min Ragan-Kelley', '[e-mail]')
            }
 
 author = 'The IPython Development Team'
 
 author_email = '[e-mail]'
 
 url = 'http://ipython.scipy.org'
 
 download_url = 'http://ipython.scipy.org/dist'
 
 platforms = ['Linux','Mac OSX','Windows XP/2000/NT','Windows 95/98/ME']
 
 keywords = ['Interactive','Interpreter','Shell','Parallel','Distributed']

IPython/external/mglob.py

@@ -1,203 +1,231 b''
 #!/usr/bin/env python
 
 r""" mglob - enhanced file list expansion module
 
 Use as stand-alone utility (for xargs, `backticks` etc.), 
 or a globbing library for own python programs. Globbing the sys.argv is something 
 that almost every Windows script has to perform manually, and this module is here
 to help with that task. Also Unix users will benefit from enhanced modes 
 such as recursion, exclusion, directory omission...
 
 Unlike glob.glob, directories are not included in the glob unless specified 
 with 'dir:'
 
 'expand' is the function to use in python programs. Typical use
 to expand argv (esp. in windows)::
 
     try:
         import mglob 
         files = mglob.expand(sys.argv[1:])
     except ImportError:
         print "mglob not found; try 'easy_install mglob' for extra features"
         files = sys.argv[1:] 
 
 Note that for unix, shell expands *normal* wildcards (*.cpp, etc.) in argv.
 Therefore, you might want to use quotes with normal wildcards to prevent this 
 expansion, in order for mglob to see the wildcards and get the wanted behaviour.
 Not quoting the wildcards is harmless and typically has equivalent results, though.
 
 Author: Ville Vainio <[e-mail]>
 License: MIT Open Source license
 
 """
 
 #Assigned in variable for "usage" printing convenience"
 
 globsyntax = """\
     This program allows specifying filenames with "mglob" mechanism.
     Supported syntax in globs (wilcard matching patterns)::
     
      *.cpp ?ellowo*                
          - obvious. Differs from normal glob in that dirs are not included.
            Unix users might want to write this as: "*.cpp" "?ellowo*"
      rec:/usr/share=*.txt,*.doc    
          - get all *.txt and *.doc under /usr/share, 
            recursively
      rec:/usr/share
          - All files under /usr/share, recursively
      rec:*.py
          - All .py files under current working dir, recursively
      foo                           
          - File or dir foo
      !*.bak readme*                   
          - readme*, exclude files ending with .bak
      !.svn/ !.hg/ !*_Data/ rec:.
          - Skip .svn, .hg, foo_Data dirs (and their subdirs) in recurse.
-           Trailing / is the key, \ does not work!
+           Trailing / is the key, \ does not work! Use !.*/ for all hidden.
      dir:foo                       
          - the directory foo if it exists (not files in foo)
      dir:*                         
          - all directories in current folder
      foo.py bar.* !h* rec:*.py
          - Obvious. !h* exclusion only applies for rec:*.py.
            foo.py is *not* included twice.
      @filelist.txt
          - All files listed in 'filelist.txt' file, on separate lines.
+     "cont:class \wak:" rec:*.py
+         - Match files containing regexp. Applies to subsequent files.
+           note quotes because of whitespace.
  """
 
 
 __version__ = "0.2"
 
 
-import os,glob,fnmatch,sys
+import os,glob,fnmatch,sys,re
 from sets import Set as set
 
                 
 def expand(flist,exp_dirs = False):
     """ Expand the glob(s) in flist.
     
     flist may be either a whitespace-separated list of globs/files
     or an array of globs/files.
     
     if exp_dirs is true, directory names in glob are expanded to the files
     contained in them - otherwise, directory names are returned as is.
     
     """
     if isinstance(flist, basestring):
-        flist = flist.split()
+        import shlex
+        flist = shlex.split(flist)
     done_set = set()
     denied_set = set()
-
+    cont_set = set()
+    cur_rejected_dirs = set()
+    
     def recfind(p, pats = ["*"]):
-        denied_dirs = ["*" + d+"*" for d in denied_set if d.endswith("/")]
-        #print "de", denied_dirs
+        denied_dirs = [os.path.dirname(d) for d in denied_set if d.endswith("/")]
         for (dp,dnames,fnames) in os.walk(p):
             # see if we should ignore the whole directory
             dp_norm = dp.replace("\\","/") + "/"
             deny = False
+            # do not traverse under already rejected dirs
+            for d in cur_rejected_dirs:
+                if dp.startswith(d):
+                    deny = True
+                    break
+            if deny:
+                continue
+            
+
             #print "dp",dp
+            bname = os.path.basename(dp)
             for deny_pat in denied_dirs:
-                if fnmatch.fnmatch( dp_norm, deny_pat):
+                if fnmatch.fnmatch( bname, deny_pat):
                     deny = True
+                    cur_rejected_dirs.add(dp)
                     break
             if deny:
                 continue
 
                     
             for f in fnames:
                 matched = False
                 for p in pats:
                     if fnmatch.fnmatch(f,p):
                         matched = True
                         break
                 if matched:
                     yield os.path.join(dp,f)            
 
     def once_filter(seq):
         for it in seq:
             p = os.path.abspath(it)
             if p in done_set:
                 continue
             done_set.add(p)
             deny = False
             for deny_pat in denied_set:
                 if fnmatch.fnmatch(os.path.basename(p), deny_pat):
                     deny = True
                     break
+            if cont_set:
+                try:
+                    cont = open(p).read()
+                except IOError:
+                    # deny
+                    continue
+                for pat in cont_set:
+                    if not re.search(pat,cont, re.IGNORECASE):
+                        deny = True
+                        break
+                    
             if not deny:
                 yield it
         return
             
     res = []
 
     for ent in flist:
         ent = os.path.expanduser(os.path.expandvars(ent))
         if ent.lower().startswith('rec:'):
             fields = ent[4:].split('=')            
             if len(fields) == 2:
                 pth, patlist = fields
             elif len(fields) == 1:
                 if os.path.isdir(fields[0]):
                     # single arg is dir
                     pth, patlist = fields[0], '*'
                 else: 
                     # single arg is pattern
                     pth, patlist = '.', fields[0]
                     
             elif len(fields) == 0:
                 pth, pathlist = '.','*'
                 
             pats = patlist.split(',')
             res.extend(once_filter(recfind(pth, pats)))
         # filelist
         elif ent.startswith('@') and os.path.isfile(ent[1:]):
             res.extend(once_filter(open(ent[1:]).read().splitlines()))
         # exclusion
         elif ent.startswith('!'):
             denied_set.add(ent[1:])
         # glob only dirs
         elif ent.lower().startswith('dir:'):
             res.extend(once_filter(filter(os.path.isdir,glob.glob(ent[4:]))))
-            
+        elif ent.lower().startswith('cont:'):
+            cont_set.add(ent[5:])
         # get all files in the specified dir
         elif os.path.isdir(ent) and exp_dirs:
             res.extend(once_filter(filter(os.path.isfile,glob.glob(ent + os.sep+"*"))))
             
         # glob only files
 
         elif '*' in ent or '?' in ent:
             res.extend(once_filter(filter(os.path.isfile,glob.glob(ent))))
 
         else:
             res.extend(once_filter([ent]))
     return res
             
             
 def test():
     assert (
         expand("*.py ~/.ipython/*.py rec:/usr/share/doc-base") == 
         expand( ['*.py', '~/.ipython/*.py', 'rec:/usr/share/doc-base'] ) 
         )
     
 def main():
     if len(sys.argv) < 2:
         print globsyntax
         return
     
     print "\n".join(expand(sys.argv[1:])),
 
 def mglob_f(self, arg):
     from IPython.genutils import SList
     if arg.strip():
         return SList(expand(arg))
     print "Please specify pattern!"
     print globsyntax
 
 def init_ipython(ip):
     """ register %mglob for IPython """
     mglob_f.__doc__ = globsyntax
     ip.expose_magic("mglob",mglob_f)  
     
 # test()
 if __name__ == "__main__":
     main()

IPython/frontend/cocoa/cocoa_frontend.py

@@ -1,425 +1,505 b''
 # encoding: utf-8
 # -*- test-case-name: IPython.frontend.cocoa.tests.test_cocoa_frontend -*-
 
 """PyObjC classes to provide a Cocoa frontend to the 
 IPython.kernel.engineservice.IEngineBase.
 
 To add an IPython interpreter to a cocoa app, instantiate an 
 IPythonCocoaController in a XIB and connect its textView outlet to an
 NSTextView instance in your UI. That's it.
 
 Author: Barry Wark
 """
 
 __docformat__ = "restructuredtext en"
 
 #-----------------------------------------------------------------------------
 #       Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
+import sys
 import objc
 import uuid
 
 from Foundation import NSObject, NSMutableArray, NSMutableDictionary,\
                         NSLog, NSNotificationCenter, NSMakeRange,\
-                        NSLocalizedString, NSIntersectionRange
+                        NSLocalizedString, NSIntersectionRange,\
+                        NSString, NSAutoreleasePool
                         
 from AppKit import NSApplicationWillTerminateNotification, NSBeep,\
                     NSTextView, NSRulerView, NSVerticalRuler
 
 from pprint import saferepr
 
 import IPython
 from IPython.kernel.engineservice import ThreadedEngineService
-from IPython.frontend.frontendbase import FrontEndBase
+from IPython.frontend.frontendbase import AsyncFrontEndBase
 
 from twisted.internet.threads import blockingCallFromThread
 from twisted.python.failure import Failure
 
 #------------------------------------------------------------------------------
 # Classes to implement the Cocoa frontend
 #------------------------------------------------------------------------------
 
 # TODO: 
 #   1. use MultiEngineClient and out-of-process engine rather than 
 #       ThreadedEngineService?
 #   2. integrate Xgrid launching of engines
         
+class AutoreleasePoolWrappedThreadedEngineService(ThreadedEngineService):
+    """Wrap all blocks in an NSAutoreleasePool"""
     
+    def wrapped_execute(self, msg, lines):
+        """wrapped_execute"""
+        try:
+            p = NSAutoreleasePool.alloc().init()
+            result = self.shell.execute(lines)
+        except Exception,e:
+            # This gives the following:
+            # et=exception class
+            # ev=exception class instance
+            # tb=traceback object
+            et,ev,tb = sys.exc_info()
+            # This call adds attributes to the exception value
+            et,ev,tb = self.shell.formatTraceback(et,ev,tb,msg)
+            # Add another attribute
+            
+            # Create a new exception with the new attributes
+            e = et(ev._ipython_traceback_text)
+            e._ipython_engine_info = msg
+            
+            # Re-raise
+            raise e
+        finally:
+            p.drain()
+        
+        return result
+    
+    def execute(self, lines):
+        # Only import this if we are going to use this class
+        from twisted.internet import threads
+        
+        msg = {'engineid':self.id,
+               'method':'execute',
+               'args':[lines]}
+        
+        d = threads.deferToThread(self.wrapped_execute, msg, lines)
+        d.addCallback(self.addIDToResult)
+        return d
 
 
-class IPythonCocoaController(NSObject, FrontEndBase):
+class IPythonCocoaController(NSObject, AsyncFrontEndBase):
     userNS = objc.ivar() #mirror of engine.user_ns (key=>str(value))
     waitingForEngine = objc.ivar().bool()
     textView = objc.IBOutlet()
     
     def init(self):
         self = super(IPythonCocoaController, self).init()
-        FrontEndBase.__init__(self, engine=ThreadedEngineService())
+        AsyncFrontEndBase.__init__(self,
+                    engine=AutoreleasePoolWrappedThreadedEngineService())
         if(self != None):
             self._common_init()
         
         return self
     
     def _common_init(self):
         """_common_init"""
         
         self.userNS = NSMutableDictionary.dictionary()
         self.waitingForEngine = False
         
         self.lines = {}
         self.tabSpaces = 4
         self.tabUsesSpaces = True
         self.currentBlockID = self.next_block_ID()
         self.blockRanges = {} # blockID=>NSRange
     
     
     def awakeFromNib(self):
         """awakeFromNib"""
         
         self._common_init()
         
         # Start the IPython engine
         self.engine.startService()
         NSLog('IPython engine started')
         
         # Register for app termination
         nc = NSNotificationCenter.defaultCenter()
         nc.addObserver_selector_name_object_(
                                     self,
                                     'appWillTerminate:',
                                     NSApplicationWillTerminateNotification,
                                     None)
         
         self.textView.setDelegate_(self)
         self.textView.enclosingScrollView().setHasVerticalRuler_(True)
         r = NSRulerView.alloc().initWithScrollView_orientation_(
                                         self.textView.enclosingScrollView(),
                                         NSVerticalRuler)
         self.verticalRulerView = r
         self.verticalRulerView.setClientView_(self.textView)
         self._start_cli_banner()
     
     
     def appWillTerminate_(self, notification):
         """appWillTerminate"""
         
         self.engine.stopService()
     
     
     def complete(self, token):
         """Complete token in engine's user_ns
         
         Parameters
         ----------
         token : string
         
         Result
         ------
         Deferred result of 
         IPython.kernel.engineservice.IEngineBase.complete
         """
         
         return self.engine.complete(token)
     
     
     def execute(self, block, blockID=None):
         self.waitingForEngine = True
         self.willChangeValueForKey_('commandHistory')
-        d = super(IPythonCocoaController, self).execute(block, blockID)
+        d = super(IPythonCocoaController, self).execute(block,
+                                                        blockID)
         d.addBoth(self._engine_done)
         d.addCallback(self._update_user_ns)
         
         return d
     
+    
+    def push_(self, namespace):
+        """Push dictionary of key=>values to python namespace"""
         
+        self.waitingForEngine = True
+        self.willChangeValueForKey_('commandHistory')
+        d = self.engine.push(namespace)
+        d.addBoth(self._engine_done)
+        d.addCallback(self._update_user_ns)
+    
+    
+    def pull_(self, keys):
+        """Pull keys from python namespace"""
+        
+        self.waitingForEngine = True
+        result = blockingCallFromThread(self.engine.pull, keys)
+        self.waitingForEngine = False
+    
+    @objc.signature('v@:@I')
+    def executeFileAtPath_encoding_(self, path, encoding):
+        """Execute file at path in an empty namespace. Update the engine
+        user_ns with the resulting locals."""
+        
+        lines,err = NSString.stringWithContentsOfFile_encoding_error_(
+            path,
+            encoding,
+            None)
+        self.engine.execute(lines)
+    
+    
     def _engine_done(self, x):
         self.waitingForEngine = False
         self.didChangeValueForKey_('commandHistory')
         return x
     
     def _update_user_ns(self, result):
         """Update self.userNS from self.engine's namespace"""
         d = self.engine.keys()
         d.addCallback(self._get_engine_namespace_values_for_keys)
         
         return result
     
     
     def _get_engine_namespace_values_for_keys(self, keys):
         d = self.engine.pull(keys)
         d.addCallback(self._store_engine_namespace_values, keys=keys)
     
     
     def _store_engine_namespace_values(self, values, keys=[]):
         assert(len(values) == len(keys))
         self.willChangeValueForKey_('userNS')
         for (k,v) in zip(keys,values):
             self.userNS[k] = saferepr(v)
         self.didChangeValueForKey_('userNS')
     
     
-    def update_cell_prompt(self, result):
+    def update_cell_prompt(self, result, blockID=None):
         if(isinstance(result, Failure)):
-            blockID = result.blockID
+            self.insert_text(self.input_prompt(),
+                textRange=NSMakeRange(self.blockRanges[blockID].location,0),
+                scrollToVisible=False
+                )
         else:
-            blockID = result['blockID']
-        
-        
-        self.insert_text(self.input_prompt(result=result),
+            self.insert_text(self.input_prompt(number=result['number']),
                 textRange=NSMakeRange(self.blockRanges[blockID].location,0),
                 scrollToVisible=False
                 )
         
         return result
     
     
     def render_result(self, result):
         blockID = result['blockID']
         inputRange = self.blockRanges[blockID]
         del self.blockRanges[blockID]
         
         #print inputRange,self.current_block_range()
         self.insert_text('\n' +
-                self.output_prompt(result) +
+                self.output_prompt(number=result['number']) +
                 result.get('display',{}).get('pprint','') +
                 '\n\n',
                 textRange=NSMakeRange(inputRange.location+inputRange.length,
                                     0))
         return result
     
         
     def render_error(self, failure):
-        self.insert_text('\n\n'+str(failure)+'\n\n')
+        self.insert_text('\n' +
+                        self.output_prompt() +
+                        '\n' +
+                        failure.getErrorMessage() +
+                        '\n\n')
         self.start_new_block()
         return failure
     
     
     def _start_cli_banner(self):
         """Print banner"""
         
         banner = """IPython1 %s -- An enhanced Interactive Python.""" % \
                     IPython.__version__
         
         self.insert_text(banner + '\n\n')
     
     
     def start_new_block(self):
         """"""
         
         self.currentBlockID = self.next_block_ID()
     
     
     
     def next_block_ID(self):
         
         return uuid.uuid4()
     
     def current_block_range(self):
         return self.blockRanges.get(self.currentBlockID, 
                         NSMakeRange(self.textView.textStorage().length(), 
                         0))
     
     def current_block(self):
         """The current block's text"""
         
         return self.text_for_range(self.current_block_range())
     
     def text_for_range(self, textRange):
         """text_for_range"""
         
         ts = self.textView.textStorage()
         return ts.string().substringWithRange_(textRange)
     
     def current_line(self):
         block = self.text_for_range(self.current_block_range())
         block = block.split('\n')
         return block[-1]
     
     
     def insert_text(self, string=None, textRange=None, scrollToVisible=True):
         """Insert text into textView at textRange, updating blockRanges 
         as necessary
         """
         
         if(textRange == None):
             #range for end of text
             textRange = NSMakeRange(self.textView.textStorage().length(), 0) 
         
         for r in self.blockRanges.itervalues():
             intersection = NSIntersectionRange(r,textRange)
             if(intersection.length == 0): #ranges don't intersect
                 if r.location >= textRange.location:
                     r.location += len(string)
             else: #ranges intersect
                 if(r.location <= textRange.location):
                     assert(intersection.length == textRange.length)
                     r.length += textRange.length
                 else:
                     r.location += intersection.length
         
         self.textView.replaceCharactersInRange_withString_(
             textRange, string)
         self.textView.setSelectedRange_(
             NSMakeRange(textRange.location+len(string), 0))
         if(scrollToVisible):
             self.textView.scrollRangeToVisible_(textRange)
         
     
     
     
     def replace_current_block_with_string(self, textView, string):
         textView.replaceCharactersInRange_withString_(
                                                 self.current_block_range(),
                                                 string)
         self.current_block_range().length = len(string)
         r = NSMakeRange(textView.textStorage().length(), 0)
         textView.scrollRangeToVisible_(r)
         textView.setSelectedRange_(r)
     
     
     def current_indent_string(self):
         """returns string for indent or None if no indent"""
         
-        if(len(self.current_block()) > 0):
-            lines = self.current_block().split('\n')
-            currentIndent = len(lines[-1]) - len(lines[-1])
+        return self._indent_for_block(self.current_block())
+    
+    
+    def _indent_for_block(self, block):
+        lines = block.split('\n')
+        if(len(lines) > 1):
+            currentIndent = len(lines[-1]) - len(lines[-1].lstrip())
             if(currentIndent == 0):
                 currentIndent = self.tabSpaces
         
             if(self.tabUsesSpaces):
                 result = ' ' * currentIndent
             else:
                 result = '\t' * (currentIndent/self.tabSpaces)
         else:
             result = None
         
         return result
     
     
     # NSTextView delegate methods...
     def textView_doCommandBySelector_(self, textView, selector):
         assert(textView == self.textView)
         NSLog("textView_doCommandBySelector_: "+selector)
         
         
         if(selector == 'insertNewline:'):
             indent = self.current_indent_string()
             if(indent):
                 line = indent + self.current_line()
             else:
                 line = self.current_line()
             
             if(self.is_complete(self.current_block())):
                 self.execute(self.current_block(),
                                 blockID=self.currentBlockID)
                 self.start_new_block()
                 
                 return True
             
             return False
         
         elif(selector == 'moveUp:'):
             prevBlock = self.get_history_previous(self.current_block())
             if(prevBlock != None):
                 self.replace_current_block_with_string(textView, prevBlock)
             else:
                 NSBeep()
             return True
         
         elif(selector == 'moveDown:'):
             nextBlock = self.get_history_next()
             if(nextBlock != None):
                 self.replace_current_block_with_string(textView, nextBlock)
             else:
                 NSBeep()
             return True
         
         elif(selector == 'moveToBeginningOfParagraph:'):
             textView.setSelectedRange_(NSMakeRange(
                                         self.current_block_range().location, 
                                         0))
             return True
         elif(selector == 'moveToEndOfParagraph:'):
             textView.setSelectedRange_(NSMakeRange(
                                     self.current_block_range().location + \
                                     self.current_block_range().length, 0))
             return True
         elif(selector == 'deleteToEndOfParagraph:'):
             if(textView.selectedRange().location <= \
                 self.current_block_range().location):
                 # Intersect the selected range with the current line range
                 if(self.current_block_range().length < 0):
                     self.blockRanges[self.currentBlockID].length = 0
             
                 r = NSIntersectionRange(textView.rangesForUserTextChange()[0],
                                         self.current_block_range())
                 
                 if(r.length > 0): #no intersection
                     textView.setSelectedRange_(r)
             
             return False # don't actually handle the delete
         
         elif(selector == 'insertTab:'):
             if(len(self.current_line().strip()) == 0): #only white space
                 return False
             else:
                 self.textView.complete_(self)
                 return True
         
         elif(selector == 'deleteBackward:'):
             #if we're at the beginning of the current block, ignore
             if(textView.selectedRange().location == \
                 self.current_block_range().location):
                 return True
             else:
                 self.current_block_range().length-=1
                 return False
         return False
     
     
     def textView_shouldChangeTextInRanges_replacementStrings_(self, 
         textView, ranges, replacementStrings):
         """
         Delegate method for NSTextView.
         
         Refuse change text in ranges not at end, but make those changes at 
         end.
         """
         
         assert(len(ranges) == len(replacementStrings))
         allow = True
         for r,s in zip(ranges, replacementStrings):
             r = r.rangeValue()
             if(textView.textStorage().length() > 0 and
                     r.location < self.current_block_range().location):
                 self.insert_text(s)
                 allow = False
             
             
             self.blockRanges.setdefault(self.currentBlockID, 
                                         self.current_block_range()).length +=\
                                          len(s)
         
         return allow
     
     def textView_completions_forPartialWordRange_indexOfSelectedItem_(self, 
         textView, words, charRange, index):
         try:
             ts = textView.textStorage()
             token = ts.string().substringWithRange_(charRange)
             completions = blockingCallFromThread(self.complete, token)
         except:
             completions = objc.nil
             NSBeep()
         
         return (completions,0)
     
 

IPython/frontend/cocoa/tests/test_cocoa_frontend.py

@@ -1,72 +1,91 b''
 # encoding: utf-8
 """This file contains unittests for the 
 IPython.frontend.cocoa.cocoa_frontend module.
 """
 __docformat__ = "restructuredtext en"
                                                                             
 #---------------------------------------------------------------------------
 #       Copyright (C) 2005  The IPython Development Team    
 #                                                                           
 #  Distributed under the terms of the BSD License.  The full license is in  
 #  the file COPYING, distributed as part of this software.                  
 #---------------------------------------------------------------------------
                                                                             
 #---------------------------------------------------------------------------
 # Imports                                                                   
 #---------------------------------------------------------------------------
 from IPython.kernel.core.interpreter import Interpreter                     
 import IPython.kernel.engineservice as es
 from IPython.testing.util import DeferredTestCase
 from twisted.internet.defer import succeed
 from IPython.frontend.cocoa.cocoa_frontend import IPythonCocoaController
                                                     
 from Foundation import NSMakeRect
 from AppKit import NSTextView, NSScrollView                                                
 
 class TestIPythonCocoaControler(DeferredTestCase):
     """Tests for IPythonCocoaController"""
     
     def setUp(self):
         self.controller = IPythonCocoaController.alloc().init()
         self.engine = es.EngineService()
         self.engine.startService()
     
     
     def tearDown(self):
         self.controller = None
         self.engine.stopService()
     
     def testControllerExecutesCode(self):
         code ="""5+5"""
         expected = Interpreter().execute(code)
         del expected['number']
         def removeNumberAndID(result):
             del result['number']
             del result['id']
             return result
         self.assertDeferredEquals(
             self.controller.execute(code).addCallback(removeNumberAndID), 
             expected)
     
     def testControllerMirrorsUserNSWithValuesAsStrings(self):
         code = """userns1=1;userns2=2"""
         def testControllerUserNS(result):
             self.assertEquals(self.controller.userNS['userns1'], 1)
             self.assertEquals(self.controller.userNS['userns2'], 2)
         
         self.controller.execute(code).addCallback(testControllerUserNS)
     
     
     def testControllerInstantiatesIEngine(self):
         self.assert_(es.IEngineBase.providedBy(self.controller.engine))
     
     def testControllerCompletesToken(self):
         code = """longNameVariable=10"""
         def testCompletes(result):
             self.assert_("longNameVariable" in result)
         
         def testCompleteToken(result):
             self.controller.complete("longNa").addCallback(testCompletes)
         
         self.controller.execute(code).addCallback(testCompletes)
     
+    
+    def testCurrentIndent(self):
+        """test that current_indent_string returns current indent or None.
+        Uses _indent_for_block for direct unit testing.
+        """
+        
+        self.controller.tabUsesSpaces = True
+        self.assert_(self.controller._indent_for_block("""a=3""") == None)
+        self.assert_(self.controller._indent_for_block("") == None)
+        block = """def test():\n    a=3"""
+        self.assert_(self.controller._indent_for_block(block) == \
+                    ' ' * self.controller.tabSpaces)
+        
+        block = """if(True):\n%sif(False):\n%spass""" % \
+                    (' '*self.controller.tabSpaces,
+                     2*' '*self.controller.tabSpaces)
+        self.assert_(self.controller._indent_for_block(block) == \
+                    2*(' '*self.controller.tabSpaces))
+    

IPython/frontend/frontendbase.py

@@ -1,352 +1,407 b''
 # encoding: utf-8
 # -*- test-case-name: IPython.frontend.tests.test_frontendbase -*-
 """
 frontendbase provides an interface and base class for GUI frontends for 
 IPython.kernel/IPython.kernel.core.
 
 Frontend implementations will likely want to subclass FrontEndBase. 
 
 Author: Barry Wark
 """
 __docformat__ = "restructuredtext en"
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
 import string
 import uuid
 import _ast
 
-import zope.interface as zi
+try:
+    from zope.interface import Interface, Attribute, implements, classProvides
+except ImportError:
+    #zope.interface is not available
+    Interface = object
+    def Attribute(name, doc): pass
+    def implements(interface): pass
+    def classProvides(interface): pass
 
 from IPython.kernel.core.history import FrontEndHistory
 from IPython.kernel.core.util import Bunch
 from IPython.kernel.engineservice import IEngineCore
 
-from twisted.python.failure import Failure
+try:
+    from twisted.python.failure import Failure
+except ImportError:
+    #Twisted not available
+    Failure = Exception
 
 ##############################################################################
 # TEMPORARY!!! fake configuration, while we decide whether to use tconfig or
 # not
 
 rc = Bunch()
 rc.prompt_in1 = r'In [$number]:  '
 rc.prompt_in2 = r'...'
 rc.prompt_out = r'Out [$number]:  '
 
 ##############################################################################
 
-class IFrontEndFactory(zi.Interface):
+class IFrontEndFactory(Interface):
     """Factory interface for frontends."""
     
     def __call__(engine=None, history=None):
         """
         Parameters:
         interpreter : IPython.kernel.engineservice.IEngineCore
         """
         
         pass
 
 
 
-class IFrontEnd(zi.Interface):
+class IFrontEnd(Interface):
     """Interface for frontends. All methods return t.i.d.Deferred"""
     
-    zi.Attribute("input_prompt_template", "string.Template instance\
+    Attribute("input_prompt_template", "string.Template instance\
                 substituteable with execute result.")
-    zi.Attribute("output_prompt_template", "string.Template instance\
+    Attribute("output_prompt_template", "string.Template instance\
                 substituteable with execute result.")
-    zi.Attribute("continuation_prompt_template", "string.Template instance\
+    Attribute("continuation_prompt_template", "string.Template instance\
                 substituteable with execute result.")
     
-    def update_cell_prompt(self, result):
+    def update_cell_prompt(result, blockID=None):
         """Subclass may override to update the input prompt for a block. 
         Since this method will be called as a 
-        twisted.internet.defer.Deferred's callback,
+        twisted.internet.defer.Deferred's callback/errback,
         implementations should return result when finished.
         
-        NB: result is a failure if the execute returned a failre. 
-        To get the blockID, you should do something like::
-            if(isinstance(result, twisted.python.failure.Failure)):
-                blockID = result.blockID
-            else:
-                blockID = result['blockID']
+        Result is a result dict in case of success, and a
+        twisted.python.util.failure.Failure in case of an error
         """
         
         pass
     
-    def render_result(self, result):
+    
+    def render_result(result):
         """Render the result of an execute call. Implementors may choose the
          method of rendering.
         For example, a notebook-style frontend might render a Chaco plot 
         inline.
         
         Parameters:
             result : dict (result of IEngineBase.execute )
+                blockID = result['blockID']
         
         Result:
             Output of frontend rendering
         """
         
         pass
     
-    def render_error(self, failure):
+    def render_error(failure):
         """Subclasses must override to render the failure. Since this method 
-        ill be called as a twisted.internet.defer.Deferred's callback, 
+        will be called as a twisted.internet.defer.Deferred's callback, 
         implementations should return result when finished.
+        
+        blockID = failure.blockID
         """
         
         pass
     
     
-    def input_prompt(result={}):
+    def input_prompt(number=''):
         """Returns the input prompt by subsituting into 
         self.input_prompt_template
         """
         pass
     
-    def output_prompt(result):
+    def output_prompt(number=''):
         """Returns the output prompt by subsituting into 
         self.output_prompt_template
         """
         
         pass
     
     def continuation_prompt():
         """Returns the continuation prompt by subsituting into 
         self.continuation_prompt_template
         """
         
         pass
     
     def is_complete(block):
         """Returns True if block is complete, False otherwise."""
         
         pass
     
     def compile_ast(block):
         """Compiles block to an _ast.AST"""
         
         pass
     
     
     def get_history_previous(currentBlock):
         """Returns the block previous in  the history. Saves currentBlock if
         the history_cursor is currently at the end of the input history"""
         pass
     
     def get_history_next():
         """Returns the next block in the history."""
         
         pass
     
 
 class FrontEndBase(object):
     """
     FrontEndBase manages the state tasks for a CLI frontend:
         - Input and output history management
         - Input/continuation and output prompt generation
         
     Some issues (due to possibly unavailable engine):
         - How do we get the current cell number for the engine?
         - How do we handle completions?
     """
     
-    zi.implements(IFrontEnd)
-    zi.classProvides(IFrontEndFactory)
-    
     history_cursor = 0
     
     current_indent_level = 0
     
     
     input_prompt_template = string.Template(rc.prompt_in1)
     output_prompt_template = string.Template(rc.prompt_out)
     continuation_prompt_template = string.Template(rc.prompt_in2)
     
-    def __init__(self, engine=None, history=None):
-        assert(engine==None or IEngineCore.providedBy(engine))
-        self.engine = IEngineCore(engine)
+    def __init__(self, shell=None, history=None):
+        self.shell = shell
         if history is None:
                 self.history = FrontEndHistory(input_cache=[''])
         else:
             self.history = history
         
     
-    def input_prompt(self, result={}):
+    def input_prompt(self, number=''):
         """Returns the current input prompt
         
         It would be great to use ipython1.core.prompts.Prompt1 here
         """
-        
-        result.setdefault('number','')
-        
-        return self.input_prompt_template.safe_substitute(result)
+        return self.input_prompt_template.safe_substitute({'number':number})
     
     
     def continuation_prompt(self):
         """Returns the current continuation prompt"""
         
         return self.continuation_prompt_template.safe_substitute()
     
-    def output_prompt(self, result):
+    def output_prompt(self, number=''):
         """Returns the output prompt for result"""
         
-        return self.output_prompt_template.safe_substitute(result)
+        return self.output_prompt_template.safe_substitute({'number':number})
     
     
     def is_complete(self, block):
         """Determine if block is complete.
         
         Parameters
         block : string
         
         Result 
         True if block can be sent to the engine without compile errors.
         False otherwise.
         """
         
         try:
             ast = self.compile_ast(block)
         except:
             return False
         
         lines = block.split('\n')
         return (len(lines)==1 or str(lines[-1])=='')
     
     
     def compile_ast(self, block):
         """Compile block to an AST
         
         Parameters:
             block : str
         
         Result:
             AST
         
         Throws:
             Exception if block cannot be compiled
         """
         
         return compile(block, "<string>", "exec", _ast.PyCF_ONLY_AST)
     
     
     def execute(self, block, blockID=None):
-        """Execute the block and return result.
+        """Execute the block and return the result.
         
         Parameters:
             block : {str, AST}
             blockID : any
                 Caller may provide an ID to identify this block. 
                 result['blockID'] := blockID
         
         Result:
             Deferred result of self.interpreter.execute
         """
         
         if(not self.is_complete(block)):
-            return Failure(Exception("Block is not compilable"))
+            raise Exception("Block is not compilable")
         
         if(blockID == None):
             blockID = uuid.uuid4() #random UUID
         
-        d = self.engine.execute(block)
-        d.addCallback(self._add_history, block=block)
-        d.addBoth(self._add_block_id, blockID)
-        d.addBoth(self.update_cell_prompt)
-        d.addCallbacks(self.render_result, errback=self.render_error)
+        try:
+            result = self.shell.execute(block)
+        except Exception,e:
+            e = self._add_block_id_for_failure(e, blockID=blockID)
+            e = self.update_cell_prompt(e, blockID=blockID)
+            e = self.render_error(e)
+        else:
+            result = self._add_block_id_for_result(result, blockID=blockID)
+            result = self.update_cell_prompt(result, blockID=blockID)
+            result = self.render_result(result)
         
-        return d
+        return result
     
     
-    def _add_block_id(self, result, blockID):
+    def _add_block_id_for_result(self, result, blockID):
         """Add the blockID to result or failure. Unfortunatley, we have to 
         treat failures differently than result dicts.
         """
         
-        if(isinstance(result, Failure)):
-            result.blockID = blockID
-        else:
-            result['blockID'] = blockID
+        result['blockID'] = blockID
         
         return result
     
+    def _add_block_id_for_failure(self, failure, blockID):
+        """_add_block_id_for_failure"""
+        
+        failure.blockID = blockID
+        return failure
+    
+    
     def _add_history(self, result, block=None):
         """Add block to the history"""
         
         assert(block != None)
         self.history.add_items([block])
         self.history_cursor += 1
         
         return result
     
     
     def get_history_previous(self, currentBlock):
         """ Returns previous history string and decrement history cursor.
         """
         command = self.history.get_history_item(self.history_cursor - 1)
         
         if command is not None:
             if(self.history_cursor == len(self.history.input_cache)):
                 self.history.input_cache[self.history_cursor] = currentBlock
             self.history_cursor -= 1
         return command
     
     
     def get_history_next(self):
         """ Returns next history string and increment history cursor.
         """
         command = self.history.get_history_item(self.history_cursor+1)
         
         if command is not None:
             self.history_cursor += 1
         return command
     
     ###
     # Subclasses probably want to override these methods...
     ###
     
-    def update_cell_prompt(self, result):
+    def update_cell_prompt(self, result, blockID=None):
         """Subclass may override to update the input prompt for a block. 
         Since this method will be called as a 
         twisted.internet.defer.Deferred's callback, implementations should 
         return result when finished.
-        
-        NB: result is a failure if the execute returned a failre. 
-        To get the blockID, you should do something like::
-            if(isinstance(result, twisted.python.failure.Failure)):
-                blockID = result.blockID
-            else:
-                blockID = result['blockID']
-            
-        
         """
         
         return result
     
     
     def render_result(self, result):
         """Subclasses must override to render result. Since this method will
         be called as a twisted.internet.defer.Deferred's callback, 
         implementations should return result when finished.
         """
         
         return result
     
     
     def render_error(self, failure):
         """Subclasses must override to render the failure. Since this method 
         will be called as a twisted.internet.defer.Deferred's callback, 
-        implementations should return result when finished."""
+        implementations should return result when finished.
+        """
         
         return failure
     
 
 
+class AsyncFrontEndBase(FrontEndBase):
+    """
+    Overrides FrontEndBase to wrap execute in a deferred result.
+    All callbacks are made as callbacks on the deferred result.
+    """
+    
+    implements(IFrontEnd)
+    classProvides(IFrontEndFactory)
+    
+    def __init__(self, engine=None, history=None):
+        assert(engine==None or IEngineCore.providedBy(engine))
+        self.engine = IEngineCore(engine)
+        if history is None:
+                self.history = FrontEndHistory(input_cache=[''])
+        else:
+            self.history = history
+    
+    
+    def execute(self, block, blockID=None):
+        """Execute the block and return the deferred result.
+        
+        Parameters:
+            block : {str, AST}
+            blockID : any
+                Caller may provide an ID to identify this block. 
+                result['blockID'] := blockID
+        
+        Result:
+            Deferred result of self.interpreter.execute
+        """
+        
+        if(not self.is_complete(block)):
+            return Failure(Exception("Block is not compilable"))
+        
+        if(blockID == None):
+            blockID = uuid.uuid4() #random UUID
+        
+        d = self.engine.execute(block)
+        d.addCallback(self._add_history, block=block)
+        d.addCallbacks(self._add_block_id_for_result,
+                errback=self._add_block_id_for_failure,
+                callbackArgs=(blockID,),
+                errbackArgs=(blockID,))
+        d.addBoth(self.update_cell_prompt, blockID=blockID)
+        d.addCallbacks(self.render_result, 
+            errback=self.render_error)
+        
+        return d
+    
+

IPython/frontend/tests/test_frontendbase.py

@@ -1,151 +1,151 b''
 # encoding: utf-8
 
 """This file contains unittests for the frontendbase module."""
 
 __docformat__ = "restructuredtext en"
 
 #---------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team                         
 #                                                                           
 #  Distributed under the terms of the BSD License.  The full license is in  
 #  the file COPYING, distributed as part of this software.                  
 #---------------------------------------------------------------------------
                                                                             
 #---------------------------------------------------------------------------
 # Imports                                                                   
 #---------------------------------------------------------------------------
 
 import unittest
 from IPython.frontend import frontendbase
 from IPython.kernel.engineservice import EngineService
 
-class FrontEndCallbackChecker(frontendbase.FrontEndBase):
+class FrontEndCallbackChecker(frontendbase.AsyncFrontEndBase):
     """FrontEndBase subclass for checking callbacks"""
     def __init__(self, engine=None, history=None):
         super(FrontEndCallbackChecker, self).__init__(engine=engine, 
                                                     history=history)
         self.updateCalled = False
         self.renderResultCalled = False
         self.renderErrorCalled = False
     
-    def update_cell_prompt(self, result):
+    def update_cell_prompt(self, result, blockID=None):
         self.updateCalled = True
         return result
     
     def render_result(self, result):
         self.renderResultCalled = True
         return result
     
     
     def render_error(self, failure):
         self.renderErrorCalled = True
         return failure
     
 
 
     
-class TestFrontendBase(unittest.TestCase):
+class TestAsyncFrontendBase(unittest.TestCase):
     def setUp(self):
         """Setup the EngineService and FrontEndBase"""
         
         self.fb = FrontEndCallbackChecker(engine=EngineService())
     
     
     def test_implements_IFrontEnd(self):
         assert(frontendbase.IFrontEnd.implementedBy(
-                                                frontendbase.FrontEndBase))
+                                    frontendbase.AsyncFrontEndBase))
     
     
     def test_is_complete_returns_False_for_incomplete_block(self):
         """"""
         
         block = """def test(a):"""
         
         assert(self.fb.is_complete(block) == False)
     
     def test_is_complete_returns_True_for_complete_block(self):
         """"""
         
         block = """def test(a): pass"""
             
         assert(self.fb.is_complete(block))
         
         block = """a=3"""
         
         assert(self.fb.is_complete(block))
     
     
     def test_blockID_added_to_result(self):
         block = """3+3"""
         
         d = self.fb.execute(block, blockID='TEST_ID')
         
         d.addCallback(self.checkBlockID, expected='TEST_ID')
     
     def test_blockID_added_to_failure(self):
         block = "raise  Exception()"
         
         d = self.fb.execute(block,blockID='TEST_ID')
         d.addErrback(self.checkFailureID, expected='TEST_ID')
     
     def checkBlockID(self, result, expected=""):
         assert(result['blockID'] == expected)
     
     
     def checkFailureID(self, failure, expected=""):
         assert(failure.blockID == expected)
     
     
     def test_callbacks_added_to_execute(self):
         """test that
         update_cell_prompt
         render_result
         
         are added to execute request
         """
         
         d = self.fb.execute("10+10")
         d.addCallback(self.checkCallbacks)
     
     
     def checkCallbacks(self, result):
         assert(self.fb.updateCalled)
         assert(self.fb.renderResultCalled)
     
     
     def test_error_callback_added_to_execute(self):
         """test that render_error called on execution error"""
         
         d = self.fb.execute("raise Exception()")
         d.addCallback(self.checkRenderError)
     
     def checkRenderError(self, result):
         assert(self.fb.renderErrorCalled)
     
     def test_history_returns_expected_block(self):
         """Make sure history browsing doesn't fail"""
         
         blocks = ["a=1","a=2","a=3"]
         for b in blocks:
             d = self.fb.execute(b)
         
         # d is now the deferred for the last executed block
         d.addCallback(self.historyTests, blocks)
         
     
     def historyTests(self, result, blocks):
         """historyTests"""
         
         assert(len(blocks) >= 3)
         assert(self.fb.get_history_previous("") == blocks[-2])
         assert(self.fb.get_history_previous("") == blocks[-3])
         assert(self.fb.get_history_next() == blocks[-2])
     
     
     def test_history_returns_none_at_startup(self):
         """test_history_returns_none_at_startup"""
         
         assert(self.fb.get_history_previous("")==None)
         assert(self.fb.get_history_next()==None)
     
     

IPython/genutils.py

@@ -1,2097 +1,2124 b''
 # -*- coding: utf-8 -*-
 """
 General purpose utilities.
 
 This is a grab-bag of stuff I find useful in most programs I write. Some of
 these things are also convenient when working at the command line.
 
 $Id: genutils.py 2998 2008-01-31 10:06:04Z vivainio $"""
 
 #*****************************************************************************
 #       Copyright (C) 2001-2006 Fernando Perez. <[e-mail]>
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #*****************************************************************************
 
 from IPython import Release
 __author__  = '%s <%s>' % Release.authors['Fernando']
 __license__ = Release.license
 
 #****************************************************************************
 # required modules from the Python standard library
 import __main__
 import commands
 try:
     import doctest
 except ImportError:
     pass
 import os
 import platform
 import re
 import shlex
 import shutil
 import subprocess
 import sys
 import tempfile
 import time
 import types
 import warnings
 
 # Curses and termios are Unix-only modules
 try:
     import curses
     # We need termios as well, so if its import happens to raise, we bail on
     # using curses altogether.
     import termios
 except ImportError:
     USE_CURSES = False
 else:
     # Curses on Solaris may not be complete, so we can't use it there
     USE_CURSES = hasattr(curses,'initscr')
 
 # Other IPython utilities
 import IPython
 from IPython.Itpl import Itpl,itpl,printpl
 from IPython import DPyGetOpt, platutils
 from IPython.generics import result_display
 import IPython.ipapi
 from IPython.external.path import path
 if os.name == "nt":
     from IPython.winconsole import get_console_size
 
 try:
     set
 except:
     from sets import Set as set
 
 
 #****************************************************************************
 # Exceptions
 class Error(Exception):
     """Base class for exceptions in this module."""
     pass
 
 #----------------------------------------------------------------------------
 class IOStream:
     def __init__(self,stream,fallback):
         if not hasattr(stream,'write') or not hasattr(stream,'flush'):
             stream = fallback
         self.stream = stream
         self._swrite = stream.write
         self.flush = stream.flush
 
     def write(self,data):
         try:
             self._swrite(data)
         except:
             try:
                 # print handles some unicode issues which may trip a plain
                 # write() call.  Attempt to emulate write() by using a
                 # trailing comma
                 print >> self.stream, data,
             except:
                 # if we get here, something is seriously broken.
                 print >> sys.stderr, \
                       'ERROR - failed to write data to stream:', self.stream
 
     def close(self):
         pass
                 
         
 class IOTerm:
     """ Term holds the file or file-like objects for handling I/O operations.
 
     These are normally just sys.stdin, sys.stdout and sys.stderr but for
     Windows they can can replaced to allow editing the strings before they are
     displayed."""
 
     # In the future, having IPython channel all its I/O operations through
     # this class will make it easier to embed it into other environments which
     # are not a normal terminal (such as a GUI-based shell)
     def __init__(self,cin=None,cout=None,cerr=None):
         self.cin  = IOStream(cin,sys.stdin)
         self.cout = IOStream(cout,sys.stdout)
         self.cerr = IOStream(cerr,sys.stderr)
         
 # Global variable to be used for all I/O
 Term = IOTerm()
 
 import IPython.rlineimpl as readline
 # Remake Term to use the readline i/o facilities
 if sys.platform == 'win32' and readline.have_readline:
     
     Term = IOTerm(cout=readline._outputfile,cerr=readline._outputfile)
     
 
 #****************************************************************************
 # Generic warning/error printer, used by everything else
 def warn(msg,level=2,exit_val=1):
     """Standard warning printer. Gives formatting consistency.
 
     Output is sent to Term.cerr (sys.stderr by default).
     
     Options:
     
     -level(2): allows finer control:
       0 -> Do nothing, dummy function.
       1 -> Print message.
       2 -> Print 'WARNING:' + message. (Default level).
       3 -> Print 'ERROR:' + message.
       4 -> Print 'FATAL ERROR:' + message and trigger a sys.exit(exit_val).
 
     -exit_val (1): exit value returned by sys.exit() for a level 4
     warning. Ignored for all other levels."""
     
     if level>0:
         header = ['','','WARNING: ','ERROR: ','FATAL ERROR: ']
         print >> Term.cerr, '%s%s' % (header[level],msg)
         if level == 4:
             print >> Term.cerr,'Exiting.\n'
             sys.exit(exit_val)
 
 def info(msg):
     """Equivalent to warn(msg,level=1)."""
 
     warn(msg,level=1)
 
 def error(msg):
     """Equivalent to warn(msg,level=3)."""
 
     warn(msg,level=3)
 
 def fatal(msg,exit_val=1):
     """Equivalent to warn(msg,exit_val=exit_val,level=4)."""
 
     warn(msg,exit_val=exit_val,level=4)
 
 #---------------------------------------------------------------------------
 # Debugging routines
 # 
 def debugx(expr,pre_msg=''):
     """Print the value of an expression from the caller's frame.
     
     Takes an expression, evaluates it in the caller's frame and prints both
     the given expression and the resulting value (as well as a debug mark
     indicating the name of the calling function.  The input must be of a form
     suitable for eval().
 
     An optional message can be passed, which will be prepended to the printed
     expr->value pair."""
     
     cf = sys._getframe(1)
     print '[DBG:%s] %s%s -> %r' % (cf.f_code.co_name,pre_msg,expr,
                                    eval(expr,cf.f_globals,cf.f_locals))
 
 # deactivate it by uncommenting the following line, which makes it a no-op
 #def debugx(expr,pre_msg=''): pass
 
 #----------------------------------------------------------------------------
 StringTypes = types.StringTypes
 
 # Basic timing functionality
 
 # If possible (Unix), use the resource module instead of time.clock()
 try:
     import resource
     def clocku():
         """clocku() -> floating point number
 
         Return the *USER* CPU time in seconds since the start of the process.
         This is done via a call to resource.getrusage, so it avoids the
         wraparound problems in time.clock()."""
         
         return resource.getrusage(resource.RUSAGE_SELF)[0]
     
     def clocks():
         """clocks() -> floating point number
 
         Return the *SYSTEM* CPU time in seconds since the start of the process.
         This is done via a call to resource.getrusage, so it avoids the
         wraparound problems in time.clock()."""
         
         return resource.getrusage(resource.RUSAGE_SELF)[1]
     
     def clock():
         """clock() -> floating point number
 
         Return the *TOTAL USER+SYSTEM* CPU time in seconds since the start of
         the process.  This is done via a call to resource.getrusage, so it
         avoids the wraparound problems in time.clock()."""
 
         u,s = resource.getrusage(resource.RUSAGE_SELF)[:2] 
         return u+s
     
     def clock2():
         """clock2() -> (t_user,t_system)
 
         Similar to clock(), but return a tuple of user/system times."""
         return resource.getrusage(resource.RUSAGE_SELF)[:2]
 
 except ImportError:
     # There is no distinction of user/system time under windows, so we just use
     # time.clock() for everything...
     clocku = clocks = clock = time.clock
     def clock2():
         """Under windows, system CPU time can't be measured.
 
         This just returns clock() and zero."""
         return time.clock(),0.0
 
 def timings_out(reps,func,*args,**kw):
     """timings_out(reps,func,*args,**kw) -> (t_total,t_per_call,output)
 
     Execute a function reps times, return a tuple with the elapsed total
     CPU time in seconds, the time per call and the function's output.
 
     Under Unix, the return value is the sum of user+system time consumed by
     the process, computed via the resource module.  This prevents problems
     related to the wraparound effect which the time.clock() function has.
     
     Under Windows the return value is in wall clock seconds. See the
     documentation for the time module for more details."""
 
     reps = int(reps)
     assert reps >=1, 'reps must be >= 1'
     if reps==1:
         start = clock()
         out = func(*args,**kw)
         tot_time = clock()-start
     else:
         rng = xrange(reps-1) # the last time is executed separately to store output
         start = clock()
         for dummy in rng: func(*args,**kw)
         out = func(*args,**kw)  # one last time
         tot_time = clock()-start
     av_time = tot_time / reps
     return tot_time,av_time,out
 
 def timings(reps,func,*args,**kw):
     """timings(reps,func,*args,**kw) -> (t_total,t_per_call)
 
     Execute a function reps times, return a tuple with the elapsed total CPU
     time in seconds and the time per call. These are just the first two values
     in timings_out()."""
 
     return timings_out(reps,func,*args,**kw)[0:2]
 
 def timing(func,*args,**kw):
     """timing(func,*args,**kw) -> t_total
 
     Execute a function once, return the elapsed total CPU time in
     seconds. This is just the first value in timings_out()."""
 
     return timings_out(1,func,*args,**kw)[0]
 
 #****************************************************************************
 # file and system
 
 def arg_split(s,posix=False):
     """Split a command line's arguments in a shell-like manner.
 
     This is a modified version of the standard library's shlex.split()
     function, but with a default of posix=False for splitting, so that quotes
     in inputs are respected."""
 
     # XXX - there may be unicode-related problems here!!!  I'm not sure that
     # shlex is truly unicode-safe, so it might be necessary to do
     #
     # s = s.encode(sys.stdin.encoding)
     #
     # first, to ensure that shlex gets a normal string.  Input from anyone who
     # knows more about unicode and shlex than I would be good to have here...
     lex = shlex.shlex(s, posix=posix)
     lex.whitespace_split = True
     return list(lex)
 
 def system(cmd,verbose=0,debug=0,header=''):
     """Execute a system command, return its exit status.
 
     Options:
 
     - verbose (0): print the command to be executed.
     
     - debug (0): only print, do not actually execute.
 
     - header (''): Header to print on screen prior to the executed command (it
     is only prepended to the command, no newlines are added).
 
     Note: a stateful version of this function is available through the
     SystemExec class."""
 
     stat = 0
     if verbose or debug: print header+cmd
     sys.stdout.flush()
     if not debug: stat = os.system(cmd)
     return stat
 
 def abbrev_cwd():
     """ Return abbreviated version of cwd, e.g. d:mydir """
     cwd = os.getcwd().replace('\\','/')
     drivepart = ''
     tail = cwd
     if sys.platform == 'win32':
         if len(cwd) < 4:
             return cwd
         drivepart,tail = os.path.splitdrive(cwd)
     
 
     parts = tail.split('/')    
     if len(parts) > 2:
         tail = '/'.join(parts[-2:])
         
     return (drivepart + (
         cwd == '/' and '/' or tail))
     
 
 # This function is used by ipython in a lot of places to make system calls.
 # We need it to be slightly different under win32, due to the vagaries of
 # 'network shares'.  A win32 override is below.
 
 def shell(cmd,verbose=0,debug=0,header=''):
     """Execute a command in the system shell, always return None.
 
     Options:
 
     - verbose (0): print the command to be executed.
     
     - debug (0): only print, do not actually execute.
 
     - header (''): Header to print on screen prior to the executed command (it
     is only prepended to the command, no newlines are added).
 
     Note: this is similar to genutils.system(), but it returns None so it can
     be conveniently used in interactive loops without getting the return value
     (typically 0) printed many times."""
 
     stat = 0
     if verbose or debug: print header+cmd
     # flush stdout so we don't mangle python's buffering
     sys.stdout.flush()
     
     if not debug:
         platutils.set_term_title("IPy " + cmd)
         os.system(cmd)
         platutils.set_term_title("IPy " + abbrev_cwd())
 
 # override shell() for win32 to deal with network shares
 if os.name in ('nt','dos'):
 
     shell_ori = shell
 
     def shell(cmd,verbose=0,debug=0,header=''):
         if os.getcwd().startswith(r"\\"):
             path = os.getcwd()
             # change to c drive (cannot be on UNC-share when issuing os.system,
             # as cmd.exe cannot handle UNC addresses)
             os.chdir("c:")
             # issue pushd to the UNC-share and then run the command
             try:
                 shell_ori('"pushd %s&&"'%path+cmd,verbose,debug,header)
             finally:
                 os.chdir(path)
         else:
             shell_ori(cmd,verbose,debug,header)
 
     shell.__doc__ = shell_ori.__doc__
 
 def getoutput(cmd,verbose=0,debug=0,header='',split=0):
     """Dummy substitute for perl's backquotes.
 
     Executes a command and returns the output.
 
     Accepts the same arguments as system(), plus:
 
     - split(0): if true, the output is returned as a list split on newlines.
 
     Note: a stateful version of this function is available through the
     SystemExec class.
     
     This is pretty much deprecated and rarely used, 
     genutils.getoutputerror may be what you need.
     
     """
 
     if verbose or debug: print header+cmd
     if not debug:
         output = os.popen(cmd).read()
         # stipping last \n is here for backwards compat.
         if output.endswith('\n'):
             output = output[:-1]
         if split:
             return output.split('\n')
         else:
             return output
 
 def getoutputerror(cmd,verbose=0,debug=0,header='',split=0):
     """Return (standard output,standard error) of executing cmd in a shell.
 
     Accepts the same arguments as system(), plus:
 
     - split(0): if true, each of stdout/err is returned as a list split on
     newlines.
 
     Note: a stateful version of this function is available through the
     SystemExec class."""
 
     if verbose or debug: print header+cmd
     if not cmd:
         if split:
             return [],[]
         else:
             return '',''
     if not debug:
         pin,pout,perr = os.popen3(cmd)
         tout = pout.read().rstrip()
         terr = perr.read().rstrip()
         pin.close()
         pout.close()
         perr.close()
         if split:
             return tout.split('\n'),terr.split('\n')
         else:
             return tout,terr
 
 # for compatibility with older naming conventions
 xsys = system
 bq = getoutput
 
 class SystemExec:
     """Access the system and getoutput functions through a stateful interface.
 
     Note: here we refer to the system and getoutput functions from this
     library, not the ones from the standard python library.
     
     This class offers the system and getoutput functions as methods, but the
     verbose, debug and header parameters can be set for the instance (at
     creation time or later) so that they don't need to be specified on each
     call.
 
     For efficiency reasons, there's no way to override the parameters on a
     per-call basis other than by setting instance attributes. If you need
     local overrides, it's best to directly call system() or getoutput().
 
     The following names are provided as alternate options:
      - xsys: alias to system
      - bq: alias to getoutput
 
     An instance can then be created as:
      >>> sysexec = SystemExec(verbose=1,debug=0,header='Calling: ')
 
     And used as:
      >>> sysexec.xsys('pwd')
      >>> dirlist = sysexec.bq('ls -l')
     """
     
     def __init__(self,verbose=0,debug=0,header='',split=0):
         """Specify the instance's values for verbose, debug and header."""
         setattr_list(self,'verbose debug header split')
 
     def system(self,cmd):
         """Stateful interface to system(), with the same keyword parameters."""
 
         system(cmd,self.verbose,self.debug,self.header)
 
     def shell(self,cmd):
         """Stateful interface to shell(), with the same keyword parameters."""
 
         shell(cmd,self.verbose,self.debug,self.header)
 
     xsys = system  # alias
 
     def getoutput(self,cmd):
         """Stateful interface to getoutput()."""
 
         return getoutput(cmd,self.verbose,self.debug,self.header,self.split)
 
     def getoutputerror(self,cmd):
         """Stateful interface to getoutputerror()."""
 
         return getoutputerror(cmd,self.verbose,self.debug,self.header,self.split)
 
     bq = getoutput  # alias
 
 #-----------------------------------------------------------------------------
 def mutex_opts(dict,ex_op):
     """Check for presence of mutually exclusive keys in a dict.
 
     Call: mutex_opts(dict,[[op1a,op1b],[op2a,op2b]...]"""
     for op1,op2 in ex_op:
         if op1 in dict and op2 in dict:
             raise ValueError,'\n*** ERROR in Arguments *** '\
                   'Options '+op1+' and '+op2+' are mutually exclusive.'
 
 #-----------------------------------------------------------------------------
 def get_py_filename(name):
     """Return a valid python filename in the current directory.
 
     If the given name is not a file, it adds '.py' and searches again.
     Raises IOError with an informative message if the file isn't found."""
 
     name = os.path.expanduser(name)
     if not os.path.isfile(name) and not name.endswith('.py'):
         name += '.py'
     if os.path.isfile(name):
         return name
     else:
         raise IOError,'File `%s` not found.' % name
 
 #-----------------------------------------------------------------------------
 def filefind(fname,alt_dirs = None):
     """Return the given filename either in the current directory, if it
     exists, or in a specified list of directories.
 
     ~ expansion is done on all file and directory names.
 
     Upon an unsuccessful search, raise an IOError exception."""
 
     if alt_dirs is None:
         try:
             alt_dirs = get_home_dir()
         except HomeDirError:
             alt_dirs = os.getcwd()
     search = [fname] + list_strings(alt_dirs)
     search = map(os.path.expanduser,search)
     #print 'search list for',fname,'list:',search  # dbg
     fname = search[0]
     if os.path.isfile(fname):
         return fname
     for direc in search[1:]:
         testname = os.path.join(direc,fname)
         #print 'testname',testname  # dbg
         if os.path.isfile(testname):
             return testname
     raise IOError,'File' + `fname` + \
           ' not found in current or supplied directories:' + `alt_dirs`
 
 #----------------------------------------------------------------------------
 def file_read(filename):
     """Read a file and close it.  Returns the file source."""
     fobj = open(filename,'r');
     source = fobj.read();
     fobj.close()
     return source
 
 def file_readlines(filename):
     """Read a file and close it.  Returns the file source using readlines()."""
     fobj = open(filename,'r');
     lines = fobj.readlines();
     fobj.close()
     return lines
 
 #----------------------------------------------------------------------------
 def target_outdated(target,deps):
     """Determine whether a target is out of date.
 
     target_outdated(target,deps) -> 1/0
 
     deps: list of filenames which MUST exist.
     target: single filename which may or may not exist.
 
     If target doesn't exist or is older than any file listed in deps, return
     true, otherwise return false.
     """
     try:
         target_time = os.path.getmtime(target)
     except os.error:
         return 1
     for dep in deps:
         dep_time = os.path.getmtime(dep)
         if dep_time > target_time:
             #print "For target",target,"Dep failed:",dep # dbg
             #print "times (dep,tar):",dep_time,target_time # dbg
             return 1
     return 0
 
 #-----------------------------------------------------------------------------
 def target_update(target,deps,cmd):
     """Update a target with a given command given a list of dependencies.
 
     target_update(target,deps,cmd) -> runs cmd if target is outdated.
 
     This is just a wrapper around target_outdated() which calls the given
     command if target is outdated."""
 
     if target_outdated(target,deps):
         xsys(cmd)
 
 #----------------------------------------------------------------------------
 def unquote_ends(istr):
     """Remove a single pair of quotes from the endpoints of a string."""
 
     if not istr:
         return istr
     if (istr[0]=="'" and istr[-1]=="'") or \
        (istr[0]=='"' and istr[-1]=='"'):
         return istr[1:-1]
     else:
         return istr
 
 #----------------------------------------------------------------------------
 def process_cmdline(argv,names=[],defaults={},usage=''):
     """ Process command-line options and arguments.
 
     Arguments:
 
     - argv: list of arguments, typically sys.argv.
     
     - names: list of option names. See DPyGetOpt docs for details on options
     syntax.
 
     - defaults: dict of default values.
 
     - usage: optional usage notice to print if a wrong argument is passed.
 
     Return a dict of options and a list of free arguments."""
 
     getopt = DPyGetOpt.DPyGetOpt()
     getopt.setIgnoreCase(0)
     getopt.parseConfiguration(names)
 
     try:
         getopt.processArguments(argv)
     except DPyGetOpt.ArgumentError, exc:
         print usage
         warn('"%s"' % exc,level=4)
 
     defaults.update(getopt.optionValues)
     args = getopt.freeValues
     
     return defaults,args
 
 #----------------------------------------------------------------------------
 def optstr2types(ostr):
     """Convert a string of option names to a dict of type mappings.
 
     optstr2types(str) -> {None:'string_opts',int:'int_opts',float:'float_opts'}
 
     This is used to get the types of all the options in a string formatted
     with the conventions of DPyGetOpt. The 'type' None is used for options
     which are strings (they need no further conversion). This function's main
     use is to get a typemap for use with read_dict().
     """
 
     typeconv = {None:'',int:'',float:''}
     typemap = {'s':None,'i':int,'f':float}
     opt_re = re.compile(r'([\w]*)([^:=]*:?=?)([sif]?)')
 
     for w in ostr.split():
         oname,alias,otype = opt_re.match(w).groups()
         if otype == '' or alias == '!':   # simple switches are integers too
             otype = 'i'
         typeconv[typemap[otype]] += oname + ' '
     return typeconv
 
 #----------------------------------------------------------------------------
 def read_dict(filename,type_conv=None,**opt):
 
     """Read a dictionary of key=value pairs from an input file, optionally
     performing conversions on the resulting values.
 
     read_dict(filename,type_conv,**opt) -> dict
 
     Only one value per line is accepted, the format should be
      # optional comments are ignored
      key value\n
 
     Args:
 
       - type_conv: A dictionary specifying which keys need to be converted to
       which types. By default all keys are read as strings. This dictionary
       should have as its keys valid conversion functions for strings
       (int,long,float,complex, or your own).  The value for each key
       (converter) should be a whitespace separated string containing the names
       of all the entries in the file to be converted using that function. For
       keys to be left alone, use None as the conversion function (only needed
       with purge=1, see below).
 
       - opt: dictionary with extra options as below (default in parens)
 
         purge(0): if set to 1, all keys *not* listed in type_conv are purged out
         of the dictionary to be returned. If purge is going to be used, the
         set of keys to be left as strings also has to be explicitly specified
         using the (non-existent) conversion function None.
 
         fs(None): field separator. This is the key/value separator to be used
         when parsing the file. The None default means any whitespace [behavior
         of string.split()].
 
         strip(0): if 1, strip string values of leading/trailinig whitespace.
 
         warn(1): warning level if requested keys are not found in file.
           - 0: silently ignore.
           - 1: inform but proceed.
           - 2: raise KeyError exception.
 
         no_empty(0): if 1, remove keys with whitespace strings as a value.
 
         unique([]): list of keys (or space separated string) which can't be
         repeated. If one such key is found in the file, each new instance
         overwrites the previous one. For keys not listed here, the behavior is
         to make a list of all appearances.
 
     Example:
     If the input file test.ini has:
       i 3
       x 4.5
       y 5.5
       s hi ho
     Then:
 
     >>> type_conv={int:'i',float:'x',None:'s'}
     >>> read_dict('test.ini')
     {'i': '3', 's': 'hi ho', 'x': '4.5', 'y': '5.5'}
     >>> read_dict('test.ini',type_conv)
     {'i': 3, 's': 'hi ho', 'x': 4.5, 'y': '5.5'}
     >>> read_dict('test.ini',type_conv,purge=1)
     {'i': 3, 's': 'hi ho', 'x': 4.5}
     """
 
     # starting config
     opt.setdefault('purge',0)
     opt.setdefault('fs',None)  # field sep defaults to any whitespace
     opt.setdefault('strip',0)
     opt.setdefault('warn',1)
     opt.setdefault('no_empty',0)
     opt.setdefault('unique','')
     if type(opt['unique']) in StringTypes:
         unique_keys = qw(opt['unique'])
     elif type(opt['unique']) in (types.TupleType,types.ListType):
         unique_keys = opt['unique']
     else:
         raise ValueError, 'Unique keys must be given as a string, List or Tuple'
 
     dict = {}
     # first read in table of values as strings
     file = open(filename,'r')
     for line in file.readlines():
         line = line.strip()
         if len(line) and line[0]=='#': continue
         if len(line)>0:
             lsplit = line.split(opt['fs'],1)
             try:
                 key,val = lsplit
             except ValueError:
                 key,val = lsplit[0],''
             key = key.strip()
             if opt['strip']: val = val.strip()
             if val == "''" or val == '""': val = ''
             if opt['no_empty'] and (val=='' or val.isspace()):
                 continue
             # if a key is found more than once in the file, build a list
             # unless it's in the 'unique' list. In that case, last found in file
             # takes precedence. User beware.
             try:
                 if dict[key] and key in unique_keys:
                     dict[key] = val
                 elif type(dict[key]) is types.ListType:
                     dict[key].append(val)
                 else:
                     dict[key] = [dict[key],val]
             except KeyError:
                 dict[key] = val
     # purge if requested
     if opt['purge']:
         accepted_keys = qwflat(type_conv.values())
         for key in dict.keys():
             if key in accepted_keys: continue
             del(dict[key])
     # now convert if requested
     if type_conv==None: return dict
     conversions = type_conv.keys()
     try: conversions.remove(None)
     except: pass
     for convert in conversions:
         for val in qw(type_conv[convert]):
             try:
                 dict[val] = convert(dict[val])
             except KeyError,e:
                 if opt['warn'] == 0:
                     pass
                 elif opt['warn'] == 1:
                     print >>sys.stderr, 'Warning: key',val,\
                           'not found in file',filename
                 elif opt['warn'] == 2:
                     raise KeyError,e
                 else:
                     raise ValueError,'Warning level must be 0,1 or 2'
 
     return dict
 
 #----------------------------------------------------------------------------
 def flag_calls(func):
     """Wrap a function to detect and flag when it gets called.
 
     This is a decorator which takes a function and wraps it in a function with
     a 'called' attribute. wrapper.called is initialized to False.
 
     The wrapper.called attribute is set to False right before each call to the
     wrapped function, so if the call fails it remains False.  After the call
     completes, wrapper.called is set to True and the output is returned.
 
     Testing for truth in wrapper.called allows you to determine if a call to
     func() was attempted and succeeded."""
     
     def wrapper(*args,**kw):
         wrapper.called = False
         out = func(*args,**kw)
         wrapper.called = True
         return out
 
     wrapper.called = False
     wrapper.__doc__ = func.__doc__
     return wrapper
 
 #----------------------------------------------------------------------------
 def dhook_wrap(func,*a,**k):
     """Wrap a function call in a sys.displayhook controller.
 
     Returns a wrapper around func which calls func, with all its arguments and
     keywords unmodified, using the default sys.displayhook.  Since IPython
     modifies sys.displayhook, it breaks the behavior of certain systems that
     rely on the default behavior, notably doctest.
     """
 
     def f(*a,**k):
     
         dhook_s = sys.displayhook
         sys.displayhook = sys.__displayhook__
         try:
             out = func(*a,**k)
         finally:
             sys.displayhook = dhook_s
 
         return out
 
     f.__doc__ = func.__doc__
     return f
 
 #----------------------------------------------------------------------------
 def doctest_reload():
     """Properly reload doctest to reuse it interactively.
 
     This routine:
 
       - reloads doctest
       
       - resets its global 'master' attribute to None, so that multiple uses of
       the module interactively don't produce cumulative reports.
     
       - Monkeypatches its core test runner method to protect it from IPython's
       modified displayhook.  Doctest expects the default displayhook behavior
       deep down, so our modification breaks it completely.  For this reason, a
       hard monkeypatch seems like a reasonable solution rather than asking
       users to manually use a different doctest runner when under IPython."""
 
     import doctest
     reload(doctest)
     doctest.master=None
 
     try:
         doctest.DocTestRunner
     except AttributeError:
         # This is only for python 2.3 compatibility, remove once we move to
         # 2.4 only.
         pass
     else:
         doctest.DocTestRunner.run = dhook_wrap(doctest.DocTestRunner.run)
 
 #----------------------------------------------------------------------------
 class HomeDirError(Error):
     pass
 
 def get_home_dir():
     """Return the closest possible equivalent to a 'home' directory.
 
     We first try $HOME.  Absent that, on NT it's $HOMEDRIVE\$HOMEPATH.
 
     Currently only Posix and NT are implemented, a HomeDirError exception is
     raised for all other OSes. """
 
     isdir = os.path.isdir
     env = os.environ
     
     # first, check py2exe distribution root directory for _ipython.
     # This overrides all. Normally does not exist.
     
     if '\\library.zip\\' in IPython.__file__.lower():
         root, rest = IPython.__file__.lower().split('library.zip')
         if isdir(root + '_ipython'):
             os.environ["IPYKITROOT"] = root.rstrip('\\')
             return root
     
     try:
         homedir = env['HOME']
         if not isdir(homedir):
             # in case a user stuck some string which does NOT resolve to a
             # valid path, it's as good as if we hadn't foud it
             raise KeyError
         return homedir
     except KeyError:
         if os.name == 'posix':
             raise HomeDirError,'undefined $HOME, IPython can not proceed.'
         elif os.name == 'nt':
             # For some strange reason, win9x returns 'nt' for os.name.
             try:
                 homedir = os.path.join(env['HOMEDRIVE'],env['HOMEPATH'])
                 if not isdir(homedir):
                     homedir = os.path.join(env['USERPROFILE'])
                     if not isdir(homedir):
                         raise HomeDirError
                 return homedir
             except:
                 try:
                     # Use the registry to get the 'My Documents' folder.
                     import _winreg as wreg
                     key = wreg.OpenKey(wreg.HKEY_CURRENT_USER,
                                        "Software\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders")
                     homedir = wreg.QueryValueEx(key,'Personal')[0]
                     key.Close()
                     if not isdir(homedir):
                         e = ('Invalid "Personal" folder registry key '
                              'typically "My Documents".\n'
                              'Value: %s\n'
                              'This is not a valid directory on your system.' %
                              homedir)
                         raise HomeDirError(e)
                     return homedir
                 except HomeDirError:
                     raise
                 except:
                     return 'C:\\'
         elif os.name == 'dos':
             # Desperate, may do absurd things in classic MacOS. May work under DOS.
             return 'C:\\'
         else:
             raise HomeDirError,'support for your operating system not implemented.'
 
 #****************************************************************************
 # strings and text
 
 class LSString(str):
     """String derivative with a special access attributes.
 
     These are normal strings, but with the special attributes:
 
         .l (or .list) : value as list (split on newlines).
         .n (or .nlstr): original value (the string itself).
         .s (or .spstr): value as whitespace-separated string.
         .p (or .paths): list of path objects
         
     Any values which require transformations are computed only once and
     cached.
 
     Such strings are very useful to efficiently interact with the shell, which
     typically only understands whitespace-separated options for commands."""
 
     def get_list(self):
         try:
             return self.__list
         except AttributeError:
             self.__list = self.split('\n')
             return self.__list
 
     l = list = property(get_list)
 
     def get_spstr(self):
         try:
             return self.__spstr
         except AttributeError:
             self.__spstr = self.replace('\n',' ')
             return self.__spstr
 
     s = spstr = property(get_spstr)
 
     def get_nlstr(self):
         return self
 
     n = nlstr = property(get_nlstr)
 
     def get_paths(self):
         try:
             return self.__paths
         except AttributeError:
             self.__paths = [path(p) for p in self.split('\n') if os.path.exists(p)]
             return self.__paths
     
     p = paths = property(get_paths)
 
 def print_lsstring(arg):
     """ Prettier (non-repr-like) and more informative printer for LSString """
     print "LSString (.p, .n, .l, .s available). Value:"
     print arg
     
 print_lsstring = result_display.when_type(LSString)(print_lsstring)
 
 #----------------------------------------------------------------------------
 class SList(list):
     """List derivative with a special access attributes.
 
     These are normal lists, but with the special attributes:
 
         .l (or .list) : value as list (the list itself).
         .n (or .nlstr): value as a string, joined on newlines.
         .s (or .spstr): value as a string, joined on spaces.
         .p (or .paths): list of path objects
         
     Any values which require transformations are computed only once and
     cached."""
 
     def get_list(self):
         return self
 
     l = list = property(get_list)
 
     def get_spstr(self):
         try:
             return self.__spstr
         except AttributeError:
             self.__spstr = ' '.join(self)
             return self.__spstr
 
     s = spstr = property(get_spstr)
 
     def get_nlstr(self):
         try:
             return self.__nlstr
         except AttributeError:
             self.__nlstr = '\n'.join(self)
             return self.__nlstr
 
     n = nlstr = property(get_nlstr)
     
     def get_paths(self):
         try:
             return self.__paths
         except AttributeError:
             self.__paths = [path(p) for p in self if os.path.exists(p)]
             return self.__paths
     
     p = paths = property(get_paths)
 
     def grep(self, pattern, prune = False, field = None):
         """ Return all strings matching 'pattern' (a regex or callable) 
         
         This is case-insensitive. If prune is true, return all items
         NOT matching the pattern.
         
         If field is specified, the match must occur in the specified 
         whitespace-separated field.
                
         Examples::
         
             a.grep( lambda x: x.startswith('C') )
             a.grep('Cha.*log', prune=1)
             a.grep('chm', field=-1)
         """
         
         def match_target(s):
             if field is None:
                 return s
             parts = s.split()
             try:
                 tgt = parts[field]
                 return tgt
             except IndexError:
                 return ""
         
         if isinstance(pattern, basestring):
             pred = lambda x : re.search(pattern, x, re.IGNORECASE)
         else:
             pred = pattern
         if not prune:
             return SList([el for el in self if pred(match_target(el))])
         else:
             return SList([el for el in self if not pred(match_target(el))])
     def fields(self, *fields):
         """ Collect whitespace-separated fields from string list
         
         Allows quick awk-like usage of string lists.
         
         Example data (in var a, created by 'a = !ls -l')::
             -rwxrwxrwx  1 ville None      18 Dec 14  2006 ChangeLog
             drwxrwxrwx+ 6 ville None       0 Oct 24 18:05 IPython        
         
         a.fields(0) is ['-rwxrwxrwx', 'drwxrwxrwx+']
         a.fields(1,0) is ['1 -rwxrwxrwx', '6 drwxrwxrwx+']
         (note the joining by space).
         a.fields(-1) is ['ChangeLog', 'IPython']
         
         IndexErrors are ignored.
         
         Without args, fields() just split()'s the strings.
         """
         if len(fields) == 0:
             return [el.split() for el in self]
         
         res = SList()
         for el in [f.split() for f in self]:
             lineparts = []
 
             for fd in fields:
                 try:
                     lineparts.append(el[fd])
                 except IndexError:
                     pass
             if lineparts:
                 res.append(" ".join(lineparts))
             
         return res        
-                
-            
-            
+    def sort(self,field= None,  nums = False):
+        """ sort by specified fields (see fields())
+        
+        Example::
+            a.sort(1, nums = True)
+        
+        Sorts a by second field, in numerical order (so that 21 > 3)
+        
+        """ 
         
+        #decorate, sort, undecorate
+        if field is not None:
+            dsu = [[SList([line]).fields(field),  line] for line in self]
+        else:
+            dsu = [[line,  line] for line in self]
+        if nums:
+            for i in range(len(dsu)):
+                numstr = "".join([ch for ch in dsu[i][0] if ch.isdigit()])
+                try:
+                    n = int(numstr)
+                except ValueError:
+                    n = 0;
+                dsu[i][0] = n
+                
+                
+        dsu.sort()
+        return SList([t[1] for t in dsu])
 
 def print_slist(arg):
     """ Prettier (non-repr-like) and more informative printer for SList """
-    print "SList (.p, .n, .l, .s, .grep(), .fields() available). Value:"
+    print "SList (.p, .n, .l, .s, .grep(), .fields(), sort() available):"
+    if hasattr(arg,  'hideonce') and arg.hideonce:
+        arg.hideonce = False
+        return
+        
     nlprint(arg)
     
 print_slist = result_display.when_type(SList)(print_slist)
 
 
 
 #----------------------------------------------------------------------------
 def esc_quotes(strng):
     """Return the input string with single and double quotes escaped out"""
 
     return strng.replace('"','\\"').replace("'","\\'")
 
 #----------------------------------------------------------------------------
 def make_quoted_expr(s):
     """Return string s in appropriate quotes, using raw string if possible.
     
     Effectively this turns string: cd \ao\ao\
     to: r"cd \ao\ao\_"[:-1]
     
     Note the use of raw string and padding at the end to allow trailing backslash.
     
     """
     
     tail = ''
     tailpadding = ''
     raw  = ''
     if "\\" in s:
         raw = 'r'
         if s.endswith('\\'):
             tail = '[:-1]'
             tailpadding = '_'
     if '"' not in s:
         quote = '"'
     elif "'" not in s:
         quote = "'"
     elif '"""' not in s and not s.endswith('"'):
         quote = '"""'
     elif "'''" not in s and not s.endswith("'"):
         quote = "'''"
     else:
         # give up, backslash-escaped string will do
         return '"%s"' % esc_quotes(s)
     res = raw + quote + s + tailpadding + quote + tail
     return res
 
 
 #----------------------------------------------------------------------------
 def raw_input_multi(header='', ps1='==> ', ps2='..> ',terminate_str = '.'):
     """Take multiple lines of input.
 
     A list with each line of input as a separate element is returned when a
     termination string is entered (defaults to a single '.'). Input can also
     terminate via EOF (^D in Unix, ^Z-RET in Windows).
 
     Lines of input which end in \\ are joined into single entries (and a
     secondary continuation prompt is issued as long as the user terminates
     lines with \\). This allows entering very long strings which are still
     meant to be treated as single entities.
     """
 
     try:
         if header:
             header += '\n'
         lines = [raw_input(header + ps1)]
     except EOFError:
         return []
     terminate = [terminate_str]
     try:
         while lines[-1:] != terminate:
             new_line = raw_input(ps1)
             while new_line.endswith('\\'):
                 new_line = new_line[:-1] + raw_input(ps2)
             lines.append(new_line)
                 
         return lines[:-1]  # don't return the termination command
     except EOFError:
         print
         return lines
 
 #----------------------------------------------------------------------------
 def raw_input_ext(prompt='',  ps2='... '):
     """Similar to raw_input(), but accepts extended lines if input ends with \\."""
 
     line = raw_input(prompt)
     while line.endswith('\\'):
         line = line[:-1] + raw_input(ps2)
     return line
 
 #----------------------------------------------------------------------------
 def ask_yes_no(prompt,default=None):
     """Asks a question and returns a boolean (y/n) answer.
 
     If default is given (one of 'y','n'), it is used if the user input is
     empty. Otherwise the question is repeated until an answer is given.
 
     An EOF is treated as the default answer.  If there is no default, an
     exception is raised to prevent infinite loops.
 
     Valid answers are: y/yes/n/no (match is not case sensitive)."""
 
     answers = {'y':True,'n':False,'yes':True,'no':False}
     ans = None
     while ans not in answers.keys():
         try:
             ans = raw_input(prompt+' ').lower()
             if not ans:  # response was an empty string
                 ans = default
         except KeyboardInterrupt:
             pass
         except EOFError:
             if default in answers.keys():
                 ans = default
                 print
             else:
                 raise
             
     return answers[ans]
 
 #----------------------------------------------------------------------------
 def marquee(txt='',width=78,mark='*'):
     """Return the input string centered in a 'marquee'."""
     if not txt:
         return (mark*width)[:width]
     nmark = (width-len(txt)-2)/len(mark)/2
     if nmark < 0: nmark =0
     marks = mark*nmark
     return '%s %s %s' % (marks,txt,marks)
 
 #----------------------------------------------------------------------------
 class EvalDict:
     """
     Emulate a dict which evaluates its contents in the caller's frame.
 
     Usage:
     >>>number = 19
     >>>text = "python"
     >>>print "%(text.capitalize())s %(number/9.0).1f rules!" % EvalDict()
     """
 
     # This version is due to [e-mail] on c.l.py, and is basically a
     # modified (shorter) version of:
     # http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/66018 by
     # Skip Montanaro ([e-mail]).
 
     def __getitem__(self, name):
         frame = sys._getframe(1)
         return eval(name, frame.f_globals, frame.f_locals)
 
 EvalString = EvalDict  # for backwards compatibility
 #----------------------------------------------------------------------------
 def qw(words,flat=0,sep=None,maxsplit=-1):
     """Similar to Perl's qw() operator, but with some more options.
 
     qw(words,flat=0,sep=' ',maxsplit=-1) -> words.split(sep,maxsplit)
 
     words can also be a list itself, and with flat=1, the output will be
     recursively flattened. Examples:
 
     >>> qw('1 2')
     ['1', '2']
     >>> qw(['a b','1 2',['m n','p q']])
     [['a', 'b'], ['1', '2'], [['m', 'n'], ['p', 'q']]]
     >>> qw(['a b','1 2',['m n','p q']],flat=1)
     ['a', 'b', '1', '2', 'm', 'n', 'p', 'q']    """
 
     if type(words) in StringTypes:
         return [word.strip() for word in words.split(sep,maxsplit)
                 if word and not word.isspace() ]
     if flat:
         return flatten(map(qw,words,[1]*len(words)))
     return map(qw,words)
 
 #----------------------------------------------------------------------------
 def qwflat(words,sep=None,maxsplit=-1):
     """Calls qw(words) in flat mode. It's just a convenient shorthand."""
     return qw(words,1,sep,maxsplit)
 
 #----------------------------------------------------------------------------
 def qw_lol(indata):
     """qw_lol('a b') -> [['a','b']],
     otherwise it's just a call to qw().
 
     We need this to make sure the modules_some keys *always* end up as a
     list of lists."""
 
     if type(indata) in StringTypes:
         return [qw(indata)]
     else:
         return qw(indata)
 
 #-----------------------------------------------------------------------------
 def list_strings(arg):
     """Always return a list of strings, given a string or list of strings
     as input."""
 
     if type(arg) in StringTypes: return [arg]
     else: return arg
 
 #----------------------------------------------------------------------------
 def grep(pat,list,case=1):
     """Simple minded grep-like function.
     grep(pat,list) returns occurrences of pat in list, None on failure.
 
     It only does simple string matching, with no support for regexps. Use the
     option case=0 for case-insensitive matching."""
 
     # This is pretty crude. At least it should implement copying only references
     # to the original data in case it's big. Now it copies the data for output.
     out=[]
     if case:
         for term in list:
             if term.find(pat)>-1: out.append(term)
     else:
         lpat=pat.lower()
         for term in list:
             if term.lower().find(lpat)>-1: out.append(term)
 
     if len(out): return out
     else: return None
 
 #----------------------------------------------------------------------------
 def dgrep(pat,*opts):
     """Return grep() on dir()+dir(__builtins__).
 
     A very common use of grep() when working interactively."""
 
     return grep(pat,dir(__main__)+dir(__main__.__builtins__),*opts)
 
 #----------------------------------------------------------------------------
 def idgrep(pat):
     """Case-insensitive dgrep()"""
 
     return dgrep(pat,0)
 
 #----------------------------------------------------------------------------
 def igrep(pat,list):
     """Synonym for case-insensitive grep."""
 
     return grep(pat,list,case=0)
 
 #----------------------------------------------------------------------------
 def indent(str,nspaces=4,ntabs=0):
     """Indent a string a given number of spaces or tabstops.
     
     indent(str,nspaces=4,ntabs=0) -> indent str by ntabs+nspaces.
     """
     if str is None:
         return
     ind = '\t'*ntabs+' '*nspaces
     outstr = '%s%s' % (ind,str.replace(os.linesep,os.linesep+ind))
     if outstr.endswith(os.linesep+ind):
         return outstr[:-len(ind)]
     else:
         return outstr
     
 #-----------------------------------------------------------------------------
 def native_line_ends(filename,backup=1):
     """Convert (in-place) a file to line-ends native to the current OS.
 
     If the optional backup argument is given as false, no backup of the
     original file is left.  """
 
     backup_suffixes = {'posix':'~','dos':'.bak','nt':'.bak','mac':'.bak'}
 
     bak_filename = filename + backup_suffixes[os.name]
 
     original = open(filename).read()
     shutil.copy2(filename,bak_filename)
     try:
         new = open(filename,'wb')
         new.write(os.linesep.join(original.splitlines()))
         new.write(os.linesep) # ALWAYS put an eol at the end of the file
         new.close()
     except:
         os.rename(bak_filename,filename)
     if not backup:
         try:
             os.remove(bak_filename)
         except:
             pass
     
 #----------------------------------------------------------------------------
 def get_pager_cmd(pager_cmd = None):
     """Return a pager command.
 
     Makes some attempts at finding an OS-correct one."""
     
     if os.name == 'posix':
         default_pager_cmd = 'less -r'  # -r for color control sequences
     elif os.name in ['nt','dos']:
         default_pager_cmd = 'type'
 
     if pager_cmd is None:
         try:
             pager_cmd = os.environ['PAGER']
         except:
             pager_cmd = default_pager_cmd
     return pager_cmd
 
 #-----------------------------------------------------------------------------
 def get_pager_start(pager,start):
     """Return the string for paging files with an offset.
 
     This is the '+N' argument which less and more (under Unix) accept.
     """
 
     if pager in ['less','more']:
         if start:
             start_string = '+' + str(start)
         else:
             start_string = ''
     else:
         start_string = ''
     return start_string
 
 #----------------------------------------------------------------------------
 # (X)emacs on W32 doesn't like to be bypassed with msvcrt.getch()
 if os.name == 'nt' and os.environ.get('TERM','dumb') != 'emacs':
     import msvcrt
     def page_more():
         """ Smart pausing between pages
 
         @return:    True if need print more lines, False if quit
         """
         Term.cout.write('---Return to continue, q to quit--- ')
         ans = msvcrt.getch()
         if ans in ("q", "Q"):
             result = False
         else:
             result = True
         Term.cout.write("\b"*37 + " "*37 + "\b"*37)
         return result
 else:
     def page_more():
         ans = raw_input('---Return to continue, q to quit--- ')
         if ans.lower().startswith('q'):
             return False
         else:
             return True
 
 esc_re = re.compile(r"(\x1b[^m]+m)")
 
 def page_dumb(strng,start=0,screen_lines=25):
     """Very dumb 'pager' in Python, for when nothing else works.
 
     Only moves forward, same interface as page(), except for pager_cmd and
     mode."""
 
     out_ln  = strng.splitlines()[start:]
     screens = chop(out_ln,screen_lines-1)
     if len(screens) == 1:
         print >>Term.cout, os.linesep.join(screens[0])
     else:
         last_escape = ""
         for scr in screens[0:-1]:
             hunk = os.linesep.join(scr)
             print >>Term.cout, last_escape + hunk
             if not page_more():
                 return
             esc_list = esc_re.findall(hunk)
             if len(esc_list) > 0:
                 last_escape = esc_list[-1]
         print >>Term.cout, last_escape + os.linesep.join(screens[-1])
 
 #----------------------------------------------------------------------------
 def page(strng,start=0,screen_lines=0,pager_cmd = None):
     """Print a string, piping through a pager after a certain length.
 
     The screen_lines parameter specifies the number of *usable* lines of your
     terminal screen (total lines minus lines you need to reserve to show other
     information).
 
     If you set screen_lines to a number <=0, page() will try to auto-determine
     your screen size and will only use up to (screen_size+screen_lines) for
     printing, paging after that. That is, if you want auto-detection but need
     to reserve the bottom 3 lines of the screen, use screen_lines = -3, and for
     auto-detection without any lines reserved simply use screen_lines = 0.
 
     If a string won't fit in the allowed lines, it is sent through the
     specified pager command. If none given, look for PAGER in the environment,
     and ultimately default to less.
 
     If no system pager works, the string is sent through a 'dumb pager'
     written in python, very simplistic.
     """
 
     # Some routines may auto-compute start offsets incorrectly and pass a
     # negative value.  Offset to 0 for robustness.
     start = max(0,start)
 
     # first, try the hook
     ip = IPython.ipapi.get()
     if ip:
         try:
             ip.IP.hooks.show_in_pager(strng)
             return
         except IPython.ipapi.TryNext:
             pass
         
     # Ugly kludge, but calling curses.initscr() flat out crashes in emacs
     TERM = os.environ.get('TERM','dumb')
     if TERM in ['dumb','emacs'] and os.name != 'nt':
         print strng
         return
     # chop off the topmost part of the string we don't want to see
     str_lines = strng.split(os.linesep)[start:]
     str_toprint = os.linesep.join(str_lines)
     num_newlines = len(str_lines)
     len_str = len(str_toprint)
     
     # Dumb heuristics to guesstimate number of on-screen lines the string
     # takes.  Very basic, but good enough for docstrings in reasonable
     # terminals. If someone later feels like refining it, it's not hard.
     numlines = max(num_newlines,int(len_str/80)+1)
 
     if os.name == "nt":
         screen_lines_def = get_console_size(defaulty=25)[1]
     else:
         screen_lines_def = 25 # default value if we can't auto-determine
 
     # auto-determine screen size
     if screen_lines <= 0:
         if TERM=='xterm':
             use_curses = USE_CURSES
         else:
             # curses causes problems on many terminals other than xterm.
             use_curses = False
         if use_curses:
             # There is a bug in curses, where *sometimes* it fails to properly
             # initialize, and then after the endwin() call is made, the
             # terminal is left in an unusable state.  Rather than trying to
             # check everytime for this (by requesting and comparing termios
             # flags each time), we just save the initial terminal state and
             # unconditionally reset it every time.  It's cheaper than making
             # the checks.
             term_flags = termios.tcgetattr(sys.stdout)
             scr = curses.initscr()
             screen_lines_real,screen_cols = scr.getmaxyx()
             curses.endwin()
             # Restore terminal state in case endwin() didn't.
             termios.tcsetattr(sys.stdout,termios.TCSANOW,term_flags)
             # Now we have what we needed: the screen size in rows/columns
             screen_lines += screen_lines_real
             #print '***Screen size:',screen_lines_real,'lines x',\
             #screen_cols,'columns.' # dbg
         else:
             screen_lines += screen_lines_def
 
     #print 'numlines',numlines,'screenlines',screen_lines  # dbg
     if numlines <= screen_lines :
         #print '*** normal print'  # dbg
         print >>Term.cout, str_toprint
     else:
         # Try to open pager and default to internal one if that fails.
         # All failure modes are tagged as 'retval=1', to match the return
         # value of a failed system command.  If any intermediate attempt
         # sets retval to 1, at the end we resort to our own page_dumb() pager.
         pager_cmd = get_pager_cmd(pager_cmd)
         pager_cmd += ' ' + get_pager_start(pager_cmd,start)
         if os.name == 'nt':
             if pager_cmd.startswith('type'):
                 # The default WinXP 'type' command is failing on complex strings.
                 retval = 1
             else:
                 tmpname = tempfile.mktemp('.txt')
                 tmpfile = file(tmpname,'wt')
                 tmpfile.write(strng)
                 tmpfile.close()
                 cmd = "%s < %s" % (pager_cmd,tmpname)
                 if os.system(cmd):
                   retval = 1
                 else:
                   retval = None
                 os.remove(tmpname)
         else:
             try:
                 retval = None
                 # if I use popen4, things hang. No idea why.
                 #pager,shell_out = os.popen4(pager_cmd)
                 pager = os.popen(pager_cmd,'w')
                 pager.write(strng)
                 pager.close()
                 retval = pager.close()  # success returns None
             except IOError,msg:  # broken pipe when user quits
                 if msg.args == (32,'Broken pipe'):
                     retval = None
                 else:
                     retval = 1
             except OSError:
                 # Other strange problems, sometimes seen in Win2k/cygwin
                 retval = 1
         if retval is not None:
             page_dumb(strng,screen_lines=screen_lines)
 
 #----------------------------------------------------------------------------
 def page_file(fname,start = 0, pager_cmd = None):
     """Page a file, using an optional pager command and starting line.
     """
 
     pager_cmd = get_pager_cmd(pager_cmd)
     pager_cmd += ' ' + get_pager_start(pager_cmd,start)
 
     try:
         if os.environ['TERM'] in ['emacs','dumb']:
             raise EnvironmentError
         xsys(pager_cmd + ' ' + fname)
     except:
         try:
             if start > 0:
                 start -= 1
             page(open(fname).read(),start)
         except:
             print 'Unable to show file',`fname`
 
 
 #----------------------------------------------------------------------------
 def snip_print(str,width = 75,print_full = 0,header = ''):
     """Print a string snipping the midsection to fit in width.
 
     print_full: mode control:
       - 0: only snip long strings
       - 1: send to page() directly.
       - 2: snip long strings and ask for full length viewing with page()
     Return 1 if snipping was necessary, 0 otherwise."""
 
     if print_full == 1:
         page(header+str)
         return 0
 
     print header,
     if len(str) < width:
         print str
         snip = 0
     else:
         whalf = int((width -5)/2)
         print str[:whalf] + ' <...> ' + str[-whalf:]
         snip = 1
     if snip and print_full == 2:
         if raw_input(header+' Snipped. View (y/n)? [N]').lower() == 'y':
             page(str)
     return snip
 
 #****************************************************************************
 # lists, dicts and structures
 
 def belong(candidates,checklist):
     """Check whether a list of items appear in a given list of options.
 
     Returns a list of 1 and 0, one for each candidate given."""
 
     return [x in checklist for x in candidates]
 
 #----------------------------------------------------------------------------
 def uniq_stable(elems):
     """uniq_stable(elems) -> list
 
     Return from an iterable, a list of all the unique elements in the input,
     but maintaining the order in which they first appear.
 
     A naive solution to this problem which just makes a dictionary with the
     elements as keys fails to respect the stability condition, since
     dictionaries are unsorted by nature.
 
     Note: All elements in the input must be valid dictionary keys for this
     routine to work, as it internally uses a dictionary for efficiency
     reasons."""
     
     unique = []
     unique_dict = {}
     for nn in elems:
         if nn not in unique_dict:
             unique.append(nn)
             unique_dict[nn] = None
     return unique
 
 #----------------------------------------------------------------------------
 class NLprinter:
     """Print an arbitrarily nested list, indicating index numbers.
 
     An instance of this class called nlprint is available and callable as a
     function.
     
     nlprint(list,indent=' ',sep=': ') -> prints indenting each level by 'indent'
     and using 'sep' to separate the index from the value. """
 
     def __init__(self):
         self.depth = 0
     
     def __call__(self,lst,pos='',**kw):
         """Prints the nested list numbering levels."""
         kw.setdefault('indent',' ')
         kw.setdefault('sep',': ')
         kw.setdefault('start',0)
         kw.setdefault('stop',len(lst))
         # we need to remove start and stop from kw so they don't propagate
         # into a recursive call for a nested list.
         start = kw['start']; del kw['start']
         stop = kw['stop']; del kw['stop']
         if self.depth == 0 and 'header' in kw.keys():
             print kw['header']
             
         for idx in range(start,stop):
             elem = lst[idx]
             if type(elem)==type([]):
                 self.depth += 1
                 self.__call__(elem,itpl('$pos$idx,'),**kw)
                 self.depth -= 1
             else:
                 printpl(kw['indent']*self.depth+'$pos$idx$kw["sep"]$elem')
 
 nlprint = NLprinter()
 #----------------------------------------------------------------------------
 def all_belong(candidates,checklist):
     """Check whether a list of items ALL appear in a given list of options.
 
     Returns a single 1 or 0 value."""
 
     return 1-(0 in [x in checklist for x in candidates])
 
 #----------------------------------------------------------------------------
 def sort_compare(lst1,lst2,inplace = 1):
     """Sort and compare two lists.
 
     By default it does it in place, thus modifying the lists. Use inplace = 0
     to avoid that (at the cost of temporary copy creation)."""
     if not inplace:
         lst1 = lst1[:]
         lst2 = lst2[:]
     lst1.sort(); lst2.sort()
     return lst1 == lst2
 
 #----------------------------------------------------------------------------
 def mkdict(**kwargs):
     """Return a dict from a keyword list.
 
     It's just syntactic sugar for making ditcionary creation more convenient:
     # the standard way
     >>>data = { 'red' : 1, 'green' : 2, 'blue' : 3 }
     # a cleaner way
     >>>data = dict(red=1, green=2, blue=3)
 
     If you need more than this, look at the Struct() class."""
 
     return kwargs
 
 #----------------------------------------------------------------------------
 def list2dict(lst):
     """Takes a list of (key,value) pairs and turns it into a dict."""
 
     dic = {}
     for k,v in lst: dic[k] = v
     return dic
 
 #----------------------------------------------------------------------------
 def list2dict2(lst,default=''):
     """Takes a list and turns it into a dict.
     Much slower than list2dict, but more versatile. This version can take
     lists with sublists of arbitrary length (including sclars)."""
 
     dic = {}
     for elem in lst:
         if type(elem) in (types.ListType,types.TupleType):
             size = len(elem)
             if  size == 0:
                 pass
             elif size == 1:
                 dic[elem] = default
             else:
                 k,v = elem[0], elem[1:]
                 if len(v) == 1: v = v[0]
                 dic[k] = v
         else:
             dic[elem] = default
     return dic
 
 #----------------------------------------------------------------------------
 def flatten(seq):
     """Flatten a list of lists (NOT recursive, only works for 2d lists)."""
 
     return [x for subseq in seq for x in subseq]
 
 #----------------------------------------------------------------------------
 def get_slice(seq,start=0,stop=None,step=1):
     """Get a slice of a sequence with variable step. Specify start,stop,step."""
     if stop == None:
         stop = len(seq)
     item = lambda i: seq[i]
     return map(item,xrange(start,stop,step))
 
 #----------------------------------------------------------------------------
 def chop(seq,size):
     """Chop a sequence into chunks of the given size."""
     chunk = lambda i: seq[i:i+size]
     return map(chunk,xrange(0,len(seq),size))
 
 #----------------------------------------------------------------------------
 # with is a keyword as of python 2.5, so this function is renamed to withobj
 # from its old 'with' name.
 def with_obj(object, **args):
     """Set multiple attributes for an object, similar to Pascal's with.
 
     Example:
     with_obj(jim,
              born = 1960,
              haircolour = 'Brown',
              eyecolour = 'Green')
 
     Credit: Greg Ewing, in
     http://mail.python.org/pipermail/python-list/2001-May/040703.html.
 
     NOTE: up until IPython 0.7.2, this was called simply 'with', but 'with'
     has become a keyword for Python 2.5, so we had to rename it."""
 
     object.__dict__.update(args)
 
 #----------------------------------------------------------------------------
 def setattr_list(obj,alist,nspace = None):
     """Set a list of attributes for an object taken from a namespace.
 
     setattr_list(obj,alist,nspace) -> sets in obj all the attributes listed in
     alist with their values taken from nspace, which must be a dict (something
     like locals() will often do) If nspace isn't given, locals() of the
     *caller* is used, so in most cases you can omit it.
 
     Note that alist can be given as a string, which will be automatically
     split into a list on whitespace. If given as a list, it must be a list of
     *strings* (the variable names themselves), not of variables."""
 
     # this grabs the local variables from the *previous* call frame -- that is
     # the locals from the function that called setattr_list().
     # - snipped from weave.inline()
     if nspace is None:
         call_frame = sys._getframe().f_back
         nspace = call_frame.f_locals
 
     if type(alist) in StringTypes:
         alist = alist.split()
     for attr in alist:
         val = eval(attr,nspace)
         setattr(obj,attr,val)
 
 #----------------------------------------------------------------------------
 def getattr_list(obj,alist,*args):
     """getattr_list(obj,alist[, default]) -> attribute list.
 
     Get a list of named attributes for an object. When a default argument is
     given, it is returned when the attribute doesn't exist; without it, an
     exception is raised in that case.
 
     Note that alist can be given as a string, which will be automatically
     split into a list on whitespace. If given as a list, it must be a list of
     *strings* (the variable names themselves), not of variables."""
 
     if type(alist) in StringTypes:
         alist = alist.split()
     if args:
         if len(args)==1:
             default = args[0]
             return map(lambda attr: getattr(obj,attr,default),alist)
         else:
             raise ValueError,'getattr_list() takes only one optional argument'
     else:
         return map(lambda attr: getattr(obj,attr),alist)
     
 #----------------------------------------------------------------------------
 def map_method(method,object_list,*argseq,**kw):
     """map_method(method,object_list,*args,**kw) -> list
 
     Return a list of the results of applying the methods to the items of the
     argument sequence(s).  If more than one sequence is given, the method is
     called with an argument list consisting of the corresponding item of each
     sequence. All sequences must be of the same length.
 
     Keyword arguments are passed verbatim to all objects called.
 
     This is Python code, so it's not nearly as fast as the builtin map()."""
 
     out_list = []
     idx = 0
     for object in object_list:
         try:
             handler = getattr(object, method)
         except AttributeError:
             out_list.append(None)
         else:
             if argseq:
                 args = map(lambda lst:lst[idx],argseq)
                 #print 'ob',object,'hand',handler,'ar',args # dbg
                 out_list.append(handler(args,**kw))
             else:
                 out_list.append(handler(**kw))
         idx += 1
     return out_list
 
 #----------------------------------------------------------------------------
 def get_class_members(cls):
     ret = dir(cls)
     if hasattr(cls,'__bases__'):
         for base in cls.__bases__:
             ret.extend(get_class_members(base))
     return ret
 
 #----------------------------------------------------------------------------
 def dir2(obj):
     """dir2(obj) -> list of strings
 
     Extended version of the Python builtin dir(), which does a few extra
     checks, and supports common objects with unusual internals that confuse
     dir(), such as Traits and PyCrust.
 
     This version is guaranteed to return only a list of true strings, whereas
     dir() returns anything that objects inject into themselves, even if they
     are later not really valid for attribute access (many extension libraries
     have such bugs).
     """
     
     # Start building the attribute list via dir(), and then complete it
     # with a few extra special-purpose calls.
     words = dir(obj)
 
     if hasattr(obj,'__class__'):
         words.append('__class__')
         words.extend(get_class_members(obj.__class__))
     #if '__base__' in words: 1/0
 
     # Some libraries (such as traits) may introduce duplicates, we want to
     # track and clean this up if it happens
     may_have_dupes = False
 
     # this is the 'dir' function for objects with Enthought's traits
     if hasattr(obj, 'trait_names'):
         try:
             words.extend(obj.trait_names())
             may_have_dupes = True
         except TypeError:
             # This will happen if `obj` is a class and not an instance.
             pass
 
     # Support for PyCrust-style _getAttributeNames magic method.
     if hasattr(obj, '_getAttributeNames'):
         try:
             words.extend(obj._getAttributeNames())
             may_have_dupes = True
         except TypeError:
             # `obj` is a class and not an instance.  Ignore
             # this error.
             pass
 
     if may_have_dupes:
         # eliminate possible duplicates, as some traits may also
         # appear as normal attributes in the dir() call.
         words = list(set(words))
         words.sort()
 
     # filter out non-string attributes which may be stuffed by dir() calls
     # and poor coding in third-party modules
     return [w for w in words if isinstance(w, basestring)]
 
 #----------------------------------------------------------------------------
 def import_fail_info(mod_name,fns=None):
     """Inform load failure for a module."""
 
     if fns == None:
         warn("Loading of %s failed.\n" % (mod_name,))
     else:
         warn("Loading of %s from %s failed.\n" % (fns,mod_name))
 
 #----------------------------------------------------------------------------
 # Proposed popitem() extension, written as a method
 
 
 class NotGiven: pass
 
 def popkey(dct,key,default=NotGiven):
     """Return dct[key] and delete dct[key].
 
     If default is given, return it if dct[key] doesn't exist, otherwise raise
     KeyError.  """
 
     try:
         val = dct[key]
     except KeyError:
         if default is NotGiven:
             raise
         else:
             return default
     else:
         del dct[key]
         return val
 
 def wrap_deprecated(func, suggest = '<nothing>'):
     def newFunc(*args, **kwargs):
         warnings.warn("Call to deprecated function %s, use %s instead" % 
                       ( func.__name__, suggest),
                       category=DeprecationWarning,
                       stacklevel = 2)
         return func(*args, **kwargs)
     return newFunc
 
 
 def _num_cpus_unix():
     """Return the number of active CPUs on a Unix system."""
     return os.sysconf("SC_NPROCESSORS_ONLN")
 
 
 def _num_cpus_darwin():
     """Return the number of active CPUs on a Darwin system."""
     p = subprocess.Popen(['sysctl','-n','hw.ncpu'],stdout=subprocess.PIPE)
     return p.stdout.read()
 
 
 def _num_cpus_windows():
     """Return the number of active CPUs on a Windows system."""
     return os.environ.get("NUMBER_OF_PROCESSORS")
 
 
 def num_cpus():
    """Return the effective number of CPUs in the system as an integer.
 
    This cross-platform function makes an attempt at finding the total number of
    available CPUs in the system, as returned by various underlying system and
    python calls.
 
    If it can't find a sensible answer, it returns 1 (though an error *may* make
    it return a large positive number that's actually incorrect).
    """
    
    # Many thanks to the Parallel Python project (http://www.parallelpython.com)
    # for the names of the keys we needed to look up for this function.  This
    # code was inspired by their equivalent function.
 
    ncpufuncs = {'Linux':_num_cpus_unix,
                 'Darwin':_num_cpus_darwin,
                 'Windows':_num_cpus_windows,
                 # On Vista, python < 2.5.2 has a bug and returns 'Microsoft'
                 # See http://bugs.python.org/issue1082 for details.
                 'Microsoft':_num_cpus_windows,
                 }
 
    ncpufunc = ncpufuncs.get(platform.system(),
                             # default to unix version (Solaris, AIX, etc)
                             _num_cpus_unix)
 
    try:
        ncpus = max(1,int(ncpufunc()))
    except:
        ncpus = 1
    return ncpus
 
 #*************************** end of file <genutils.py> **********************

IPython/kernel/scripts/ipcluster.py

@@ -1,323 +1,324 b''
 #!/usr/bin/env python
 # encoding: utf-8
 
 """Start an IPython cluster conveniently, either locally or remotely.
 
 Basic usage
 -----------
 
 For local operation, the simplest mode of usage is:
 
   %prog -n N
 
 where N is the number of engines you want started.
 
 For remote operation, you must call it with a cluster description file:
 
   %prog -f clusterfile.py
 
 The cluster file is a normal Python script which gets run via execfile().  You
 can have arbitrary logic in it, but all that matters is that at the end of the
 execution, it declares the variables 'controller', 'engines', and optionally
 'sshx'.  See the accompanying examples for details on what these variables must
 contain.
 
 
 Notes
 -----
 
 WARNING: this code is still UNFINISHED and EXPERIMENTAL!  It is incomplete,
 some listed options are not really implemented, and all of its interfaces are
 subject to change.
 
 When operating over SSH for a remote cluster, this program relies on the
 existence of a particular script called 'sshx'.  This script must live in the
 target systems where you'll be running your controller and engines, and is
 needed to configure your PATH and PYTHONPATH variables for further execution of
 python code at the other end of an SSH connection.  The script can be as simple
 as:
 
 #!/bin/sh
 . $HOME/.bashrc
 "$@"
 
 which is the default one provided by IPython.  You can modify this or provide
 your own.  Since it's quite likely that for different clusters you may need
 this script to configure things differently or that it may live in different
 locations, its full path can be set in the same file where you define the
 cluster setup.  IPython's order of evaluation for this variable is the
 following:
 
   a) Internal default: 'sshx'.  This only works if it is in the default system
   path which SSH sets up in non-interactive mode.
 
   b) Environment variable: if $IPYTHON_SSHX is defined, this overrides the
   internal default.
 
   c) Variable 'sshx' in the cluster configuration file: finally, this will
   override the previous two values.
  
 This code is Unix-only, with precious little hope of any of this ever working
 under Windows, since we need SSH from the ground up, we background processes,
 etc.  Ports of this functionality to Windows are welcome.
 
 
 Call summary
 ------------
 
     %prog [options]
 """
 
 __docformat__ = "restructuredtext en"
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # Stdlib imports
 #-------------------------------------------------------------------------------
 
 import os
 import signal
 import sys
 import time
 
 from optparse import OptionParser
 from subprocess import Popen,call
 
 #---------------------------------------------------------------------------
 # IPython imports
 #---------------------------------------------------------------------------
 from IPython.tools import utils
 from IPython.config import cutils
 
 #---------------------------------------------------------------------------
 # Normal code begins
 #---------------------------------------------------------------------------
 
 def parse_args():
     """Parse command line and return opts,args."""
 
     parser = OptionParser(usage=__doc__)
     newopt = parser.add_option  # shorthand
 
     newopt("--controller-port", type="int", dest="controllerport",
            help="the TCP port the controller is listening on")
 
     newopt("--controller-ip", type="string", dest="controllerip",
            help="the TCP ip address of the controller")
 
     newopt("-n", "--num", type="int", dest="n",default=2,
            help="the number of engines to start")
 
     newopt("--engine-port", type="int", dest="engineport",
            help="the TCP port the controller will listen on for engine "
            "connections")
     
     newopt("--engine-ip", type="string", dest="engineip",
            help="the TCP ip address the controller will listen on "
            "for engine connections")
 
     newopt("--mpi", type="string", dest="mpi",
            help="use mpi with package:  for instance --mpi=mpi4py")
     
     newopt("-l", "--logfile", type="string", dest="logfile",
            help="log file name")
 
     newopt('-f','--cluster-file',dest='clusterfile',
            help='file describing a remote cluster')
 
     return parser.parse_args()
 
 def numAlive(controller,engines):
     """Return the number of processes still alive."""
     retcodes = [controller.poll()] + \
                [e.poll() for e in engines]
     return retcodes.count(None)
 
 stop = lambda pid: os.kill(pid,signal.SIGINT)
 kill = lambda pid: os.kill(pid,signal.SIGTERM)
 
 def cleanup(clean,controller,engines):
     """Stop the controller and engines with the given cleanup method."""
     
     for e in engines:
         if e.poll() is None:
             print 'Stopping engine, pid',e.pid
             clean(e.pid)
     if controller.poll() is None:
         print 'Stopping controller, pid',controller.pid
         clean(controller.pid)
 
 
 def ensureDir(path):
     """Ensure a directory exists or raise an exception."""
     if not os.path.isdir(path):
         os.makedirs(path)
 
 
 def startMsg(control_host,control_port=10105):
     """Print a startup message"""
     print
     print 'Your cluster is up and running.'
     print
     print 'For interactive use, you can make a MultiEngineClient with:'
     print
     print 'from IPython.kernel import client'
-    print "mec = client.MultiEngineClient((%r,%s))" % \
-          (control_host,control_port)
+    print "mec = client.MultiEngineClient()"
     print
     print 'You can then cleanly stop the cluster from IPython using:'
     print
     print 'mec.kill(controller=True)'
     print
 
     
 def clusterLocal(opt,arg):
     """Start a cluster on the local machine."""
     
     # Store all logs inside the ipython directory
     ipdir = cutils.get_ipython_dir()
     pjoin = os.path.join
 
     logfile = opt.logfile
     if logfile is None:
         logdir_base = pjoin(ipdir,'log')
         ensureDir(logdir_base)
         logfile = pjoin(logdir_base,'ipcluster-')
 
     print 'Starting controller:',
-    controller = Popen(['ipcontroller','--logfile',logfile])
+    controller = Popen(['ipcontroller','--logfile',logfile,'-x','-y'])
     print 'Controller PID:',controller.pid
 
     print 'Starting engines:   ',
-    time.sleep(3)
+    time.sleep(5)
 
     englogfile = '%s%s-' % (logfile,controller.pid)
     mpi = opt.mpi
     if mpi: # start with mpi - killing the engines with sigterm will not work if you do this
-        engines = [Popen(['mpirun', '-np', str(opt.n), 'ipengine', '--mpi', mpi, '--logfile',englogfile])]
+        engines = [Popen(['mpirun', '-np', str(opt.n), 'ipengine', '--mpi', 
+            mpi, '--logfile',englogfile])]
+        # engines = [Popen(['mpirun', '-np', str(opt.n), 'ipengine', '--mpi', mpi])]
     else: # do what we would normally do
         engines = [ Popen(['ipengine','--logfile',englogfile])
                     for i in range(opt.n) ]
     eids = [e.pid for e in engines]
     print 'Engines PIDs:  ',eids
     print 'Log files: %s*' % englogfile
     
     proc_ids = eids + [controller.pid]
     procs = engines + [controller]
 
     grpid = os.getpgrp()
     try:
         startMsg('127.0.0.1')
         print 'You can also hit Ctrl-C to stop it, or use from the cmd line:'
         print
         print 'kill -INT',grpid
         print
         try:
             while True:
                 time.sleep(5)
         except:
             pass
     finally:
         print 'Stopping cluster.  Cleaning up...'
         cleanup(stop,controller,engines)
         for i in range(4):
             time.sleep(i+2)
             nZombies = numAlive(controller,engines)
             if  nZombies== 0:
                 print 'OK: All processes cleaned up.'
                 break
             print 'Trying again, %d processes did not stop...' % nZombies
             cleanup(kill,controller,engines)
             if numAlive(controller,engines) == 0:
                 print 'OK: All processes cleaned up.'
                 break
         else:
             print '*'*75
             print 'ERROR: could not kill some processes, try to do it',
             print 'manually.'
             zombies = []
             if controller.returncode is None:
                 print 'Controller is alive: pid =',controller.pid
                 zombies.append(controller.pid)
             liveEngines = [ e for e in engines if e.returncode is None ]
             for e in liveEngines:
                 print 'Engine is alive:     pid =',e.pid
                 zombies.append(e.pid)
             print
             print 'Zombie summary:',' '.join(map(str,zombies))
 
 def clusterRemote(opt,arg):
     """Start a remote cluster over SSH"""
 
     # Load the remote cluster configuration
     clConfig = {}
     execfile(opt.clusterfile,clConfig)
     contConfig = clConfig['controller']
     engConfig = clConfig['engines']
     # Determine where to find sshx:
     sshx = clConfig.get('sshx',os.environ.get('IPYTHON_SSHX','sshx'))
     
     # Store all logs inside the ipython directory
     ipdir = cutils.get_ipython_dir()
     pjoin = os.path.join
 
     logfile = opt.logfile
     if logfile is None:
         logdir_base = pjoin(ipdir,'log')
         ensureDir(logdir_base)
         logfile = pjoin(logdir_base,'ipcluster')
 
     # Append this script's PID to the logfile name always
     logfile = '%s-%s' % (logfile,os.getpid())
     
     print 'Starting controller:'
     # Controller data:
     xsys = os.system
 
     contHost = contConfig['host']
     contLog = '%s-con-%s-' % (logfile,contHost)
     cmd = "ssh %s '%s' 'ipcontroller --logfile %s' &" % \
           (contHost,sshx,contLog)
     #print 'cmd:<%s>' % cmd  # dbg
     xsys(cmd)
     time.sleep(2)
 
     print 'Starting engines:   '
     for engineHost,engineData in engConfig.iteritems():
         if isinstance(engineData,int):
             numEngines = engineData
         else:
             raise NotImplementedError('port configuration not finished for engines')
 
         print 'Sarting %d engines on %s' % (numEngines,engineHost)
         engLog = '%s-eng-%s-' % (logfile,engineHost)
         for i in range(numEngines):
             cmd = "ssh %s '%s' 'ipengine --controller-ip %s --logfile %s' &" % \
                       (engineHost,sshx,contHost,engLog)
             #print 'cmd:<%s>' % cmd  # dbg
             xsys(cmd)
         # Wait after each host a little bit
         time.sleep(1)
             
     startMsg(contConfig['host'])
         
 def main():
     """Main driver for the two big options: local or remote cluster."""
     
     opt,arg = parse_args()
 
     clusterfile = opt.clusterfile
     if clusterfile:
         clusterRemote(opt,arg)
     else:
         clusterLocal(opt,arg)
         
             
 if __name__=='__main__':
     main()

IPython/kernel/scripts/ipengine.py

@@ -1,169 +1,171 b''
 #!/usr/bin/env python
 # encoding: utf-8
 
 """Start the IPython Engine."""
 
 __docformat__ = "restructuredtext en"
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
 
 # Python looks for an empty string at the beginning of sys.path to enable
 # importing from the cwd.
 import sys
 sys.path.insert(0, '')
 
 import sys, os
 from optparse import OptionParser
 
 from twisted.application import service
 from twisted.internet import reactor
 from twisted.python import log
 
 from IPython.kernel.fcutil import Tub, UnauthenticatedTub
 
 from IPython.kernel.core.config import config_manager as core_config_manager
 from IPython.config.cutils import import_item
 from IPython.kernel.engineservice import EngineService
 from IPython.kernel.config import config_manager as kernel_config_manager
 from IPython.kernel.engineconnector import EngineConnector
 
 
 #-------------------------------------------------------------------------------
 # Code
 #-------------------------------------------------------------------------------
 
 def start_engine():
     """
     Start the engine, by creating it and starting the Twisted reactor.
     
     This method does:
     
         * If it exists, runs the `mpi_import_statement` to call `MPI_Init`
         * Starts the engine logging
         * Creates an IPython shell and wraps it in an `EngineService`
         * Creates a `foolscap.Tub` to use in connecting to a controller.
         * Uses the tub and the `EngineService` along with a Foolscap URL
           (or FURL) to connect to the controller and register the engine
           with the controller
     """
     kernel_config = kernel_config_manager.get_config_obj()
     core_config = core_config_manager.get_config_obj()
     
+
     # Execute the mpi import statement that needs to call MPI_Init
+    global mpi
     mpikey = kernel_config['mpi']['default']
     mpi_import_statement = kernel_config['mpi'].get(mpikey, None)
     if mpi_import_statement is not None:
         try:
-            exec mpi_import_statement in locals(), globals()
+            exec mpi_import_statement in globals()
         except:
             mpi = None
     else:
         mpi = None
     
     # Start logging
     logfile = kernel_config['engine']['logfile']
     if logfile:
         logfile = logfile + str(os.getpid()) + '.log'
         try:
             openLogFile = open(logfile, 'w')
         except:
             openLogFile = sys.stdout
     else:
         openLogFile = sys.stdout
     log.startLogging(openLogFile)
     
     # Create the underlying shell class and EngineService
     shell_class = import_item(core_config['shell']['shell_class'])
     engine_service = EngineService(shell_class, mpi=mpi)
     shell_import_statement = core_config['shell']['import_statement']
     if shell_import_statement:
         try:
             engine_service.execute(shell_import_statement)
         except:
             log.msg("Error running import_statement: %s" % sis)
     
     # Create the service hierarchy
     main_service = service.MultiService()
     engine_service.setServiceParent(main_service)
     tub_service = Tub()
     tub_service.setServiceParent(main_service)
     # This needs to be called before the connection is initiated
     main_service.startService()
     
     # This initiates the connection to the controller and calls
     # register_engine to tell the controller we are ready to do work
     engine_connector = EngineConnector(tub_service)
     furl_file = kernel_config['engine']['furl_file']
     d = engine_connector.connect_to_controller(engine_service, furl_file)
     d.addErrback(lambda _: reactor.stop())
     
     reactor.run()
 
 
 def init_config():
     """
     Initialize the configuration using default and command line options.
     """
     
     parser = OptionParser()
     
     parser.add_option(
         "--furl-file",
         type="string", 
         dest="furl_file",
         help="The filename containing the FURL of the controller"
     )
     parser.add_option(
         "--mpi",
         type="string",
         dest="mpi",
         help="How to enable MPI (mpi4py, pytrilinos, or empty string to disable)"
     )
     parser.add_option(
         "-l",
         "--logfile",
         type="string",
         dest="logfile",
         help="log file name (default is stdout)"
     )
     parser.add_option(
         "--ipythondir",
         type="string",
         dest="ipythondir",
         help="look for config files and profiles in this directory"
     )
     
     (options, args) = parser.parse_args()
     
     kernel_config_manager.update_config_obj_from_default_file(options.ipythondir)
     core_config_manager.update_config_obj_from_default_file(options.ipythondir)
     
     kernel_config = kernel_config_manager.get_config_obj()
     # Now override with command line options
     if options.furl_file is not None:
         kernel_config['engine']['furl_file'] = options.furl_file
     if options.logfile is not None:
         kernel_config['engine']['logfile'] = options.logfile
     if options.mpi is not None:
         kernel_config['mpi']['default'] = options.mpi
 
 
 def main():
     """
     After creating the configuration information, start the engine.
     """
     init_config()
     start_engine()
 
 
 if __name__ == "__main__":
     main()
  No newline at end of file

IPython/platutils.py

@@ -1,35 +1,47 b''
 # -*- coding: utf-8 -*-
 """ Proxy module for accessing platform specific utility functions. 
 
 Importing this module should give you the implementations that are correct 
 for your operation system, from platutils_PLATFORMNAME module.
-
-$Id: ipstruct.py 1005 2006-01-12 08:39:26Z fperez $
-
-
 """
 
-
 #*****************************************************************************
 #       Copyright (C) 2001-2006 Fernando Perez <[e-mail]>
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #*****************************************************************************
 
 from IPython import Release
 __author__  = '%s <%s>' % Release.authors['Ville']
 __license__ = Release.license
 
-import os,sys
+import os
+import sys
 
+# Import the platform-specific implementations
 if os.name == 'posix':
-    from platutils_posix import *
+    import platutils_posix as _platutils
 elif sys.platform == 'win32':
-    from platutils_win32 import *
+    import platutils_win32 as _platutils
 else:
-    from platutils_dummy import *
+    import platutils_dummy as _platutils
     import warnings
     warnings.warn("Platutils not available for platform '%s', some features may be missing" %
         os.name)
     del warnings
+
+
+# Functionality that's logically common to all platforms goes here, each
+# platform-specific module only provides the bits that are OS-dependent.
+
+def freeze_term_title():
+    _platutils.ignore_termtitle = True
+
+
+def set_term_title(title):
+    """Set terminal title using the necessary platform-dependent calls."""
+
+    if _platutils.ignore_termtitle:
+        return
+    _platutils.set_term_title(title)

IPython/platutils_dummy.py

@@ -1,29 +1,24 b''
 # -*- coding: utf-8 -*-
 """ Platform specific utility functions, dummy version 
 
 This has empty implementation of the platutils functions, used for 
 unsupported operating systems.
-
-$Id: ipstruct.py 1005 2006-01-12 08:39:26Z fperez $
-
-
 """
 
-
 #*****************************************************************************
 #       Copyright (C) 2001-2006 Fernando Perez <[e-mail]>
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #*****************************************************************************
 
 from IPython import Release
 __author__  = '%s <%s>' % Release.authors['Ville']
 __license__ = Release.license
 
+# This variable is part of the expected API of the module:
+ignore_termtitle = True
 
-def _dummy(*args,**kw):
+def set_term_title(*args,**kw):
+    """Dummy no-op."""
     pass
-
-set_term_title = _dummy
-

IPython/platutils_posix.py

@@ -1,47 +1,36 b''
 # -*- coding: utf-8 -*-
 """ Platform specific utility functions, posix version 
 
 Importing this module directly is not portable - rather, import platutils 
 to use these functions in platform agnostic fashion.
-
-$Id: ipstruct.py 1005 2006-01-12 08:39:26Z fperez $
-
 """
 
-
 #*****************************************************************************
 #       Copyright (C) 2001-2006 Fernando Perez <[e-mail]>
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #*****************************************************************************
 
 from IPython import Release
 __author__  = '%s <%s>' % Release.authors['Ville']
 __license__ = Release.license
 
 import sys
 import os
 
 ignore_termtitle = False
 
 def _dummy_op(*a, **b):
     """ A no-op function """
 
 def _set_term_title_xterm(title):
     """ Change virtual terminal title in xterm-workalikes """
 
-    if ignore_termtitle:
-        return
-    
     sys.stdout.write('\033]%d;%s\007' % (0,title))
 
 
 if os.environ.get('TERM','') == 'xterm':
     set_term_title = _set_term_title_xterm
 else:
     set_term_title = _dummy_op
-
-def freeze_term_title():
-    global ignore_termtitle
-    ignore_termtitle = True

IPython/platutils_win32.py

@@ -1,56 +1,47 b''
 # -*- coding: utf-8 -*-
 """ Platform specific utility functions, win32 version
 
 Importing this module directly is not portable - rather, import platutils 
 to use these functions in platform agnostic fashion.
-
-$Id: ipstruct.py 1005 2006-01-12 08:39:26Z fperez $
-
 """
 
-
 #*****************************************************************************
 #       Copyright (C) 2001-2006 Fernando Perez <[e-mail]>
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #*****************************************************************************
 
 from IPython import Release
 __author__  = '%s <%s>' % Release.authors['Ville']
 __license__ = Release.license
 
 import os
 
-ignore_termtitle = 0
+ignore_termtitle = False
 
 try:
     import ctypes
-    SetConsoleTitleW=ctypes.windll.kernel32.SetConsoleTitleW
-    SetConsoleTitleW.argtypes=[ctypes.c_wchar_p]
-    def _set_term_title(title):
-        """ Set terminal title using the ctypes"""
+
+    SetConsoleTitleW = ctypes.windll.kernel32.SetConsoleTitleW
+    SetConsoleTitleW.argtypes = [ctypes.c_wchar_p]
+    
+    def set_term_title(title):
+        """Set terminal title using ctypes to access the Win32 APIs."""
         SetConsoleTitleW(title)
 
 except ImportError:
-    def _set_term_title(title):
-        """ Set terminal title using the 'title' command """
-        curr=os.getcwd()
-        os.chdir("C:") #Cannot be on network share when issuing system commands
-        ret = os.system("title " + title)
-        os.chdir(curr)
+    def set_term_title(title):
+        """Set terminal title using the 'title' command."""
+        global ignore_termtitle
+
+        try:
+            # Cannot be on network share when issuing system commands
+            curr = os.getcwd()
+            os.chdir("C:")
+            ret = os.system("title " + title)
+        finally:
+            os.chdir(curr)
         if ret:
-            ignore_termtitle = 1
-
-def set_term_title(title):
-    """ Set terminal title using the 'title' command """
-    global ignore_termtitle
-
-    if ignore_termtitle:
-        return
-    _set_term_title(title)
-
-def freeze_term_title():
-    global ignore_termtitle
-    ignore_termtitle = 1
-
+            # non-zero return code signals error, don't try again
+            ignore_termtitle = True

IPython/testing/tutils.py

@@ -1,72 +1,86 b''
-"""Utilities for testing code.
+"""DEPRECATED - use IPython.testing.util instead.
+
+Utilities for testing code.
 """
 
+#############################################################################
+
+# This was old testing code we never really used in IPython.  The pieces of
+# testing machinery from snakeoil that were good have already been merged into
+# the nose plugin, so this can be taken away soon.  Leave a warning for now,
+# we'll remove it in a later release (around 0.10 or so).
+from warnings import warn
+warn('This will be removed soon.  Use IPython.testing.util instead',
+     DeprecationWarning)
+
+#############################################################################
+
 # Required modules and packages
 
 # Standard Python lib
 import os
 import sys
 
 # From this project
 from IPython.tools import utils
 
 # path to our own installation, so we can find source files under this.
 TEST_PATH = os.path.dirname(os.path.abspath(__file__))
 
 # Global flag, used by vprint
 VERBOSE = '-v' in sys.argv or '--verbose' in sys.argv
 
 ##########################################################################
 # Code begins
 
 # Some utility functions
 def vprint(*args):
     """Print-like function which relies on a global VERBOSE flag."""
     if not VERBOSE:
         return
 
     write = sys.stdout.write
     for item in args:
         write(str(item))
     write('\n')
     sys.stdout.flush()
 
 def test_path(path):
     """Return a path as a subdir of the test package.
 
     This finds the correct path of the test package on disk, and prepends it
     to the input path."""
-    
+
     return os.path.join(TEST_PATH,path)
 
 def fullPath(startPath,files):
     """Make full paths for all the listed files, based on startPath.
 
     Only the base part of startPath is kept, since this routine is typically
     used with a script's __file__ variable as startPath.  The base of startPath
     is then prepended to all the listed files, forming the output list.
 
     :Parameters:
       startPath : string
         Initial path to use as the base for the results.  This path is split
       using os.path.split() and only its first component is kept.
 
       files : string or list
         One or more files.
 
     :Examples:
-    
+
     >>> fullPath('/foo/bar.py',['a.txt','b.txt'])
     ['/foo/a.txt', '/foo/b.txt']
 
     >>> fullPath('/foo',['a.txt','b.txt'])
     ['/a.txt', '/b.txt']
 
     If a single file is given, the output is still a list:
     >>> fullPath('/foo','a.txt')
     ['/a.txt']
     """
-    
+
     files = utils.list_strings(files)
     base = os.path.split(startPath)[0]
     return [ os.path.join(base,f) for f in files ]

IPython/tools/tests/test_tools_utils.txt

@@ -1,42 +1,17 b''
 =========================================
  Doctests for the ``tools.utils`` module
 =========================================
 
 The way doctest loads these, the entire document is applied as a single test
-rather than multiple individual ones, unfortunately.
+rather than multiple individual ones, unfortunately::
 
-
-Auto-generated tests
-====================
-
-
-----------------------------------------------------------------------------
-
-Begin included file tst_tools_utils_doctest2.py::
-
-    # Setup - all imports are done in tcommon
-    >>> from IPython.testing import tcommon
-    >>> from IPython.testing.tcommon import *
-    
-    # Doctest code begins here
     >>> from IPython.tools import utils
-    
+
     # Some other tests for utils
-    
+
     >>> utils.marquee('Testing marquee')
     '****************************** Testing marquee ******************************'
-    
+
     >>> utils.marquee('Another test',30,'.')
     '........ Another test ........'
-    
-    
-End included file tst_tools_utils_doctest2.py
-
-----------------------------------------------------------------------------
-
-
-
-Manually generated tests
-========================
 
-These are one-off tests written by hand, copied from an interactive prompt.

MANIFEST.in

@@ -1,31 +1,30 b''
 include README_Windows.txt
 include win32_manual_post_install.py
 include ipython.py
 
 graft scripts
 
 graft setupext
 
 graft IPython/UserConfig
 
 graft IPython/kernel
 graft IPython/config
 graft IPython/testing
 graft IPython/tools
 
 graft docs
 exclude docs/\#*
 exclude docs/man/*.1
-exclude docs/ChangeLog.*
 
 # There seems to be no way of excluding whole subdirectories, other than
 # manually excluding all their subdirs.  distutils really is horrible...
 exclude docs/attic/*
 exclude docs/build/*
 
 global-exclude *~
 global-exclude *.flc
 global-exclude *.pyc
 global-exclude .dircopy.log
 global-exclude .svn
 global-exclude .bzr

docs/emacs/ipython.el

@@ -1,483 +1,491 b''
 ;;; ipython.el --- Adds support for IPython to python-mode.el
 
 ;; Copyright (C) 2002, 2003, 2004, 2005 Alexander Schmolck
 ;; Author:        Alexander Schmolck
 ;; Keywords:      ipython python languages oop
 ;; URL:           http://ipython.scipy.org
 ;; Compatibility: Emacs21, XEmacs21
 ;; FIXME: #$@! INPUT RING
 (defconst ipython-version "$Revision: 2927 $"
   "VC version number.")
 
-;;; Commentary 
+;;; Commentary
 ;; This library makes all the functionality python-mode has when running with
 ;; the normal python-interpreter available for ipython, too. It also enables a
 ;; persistent py-shell command history across sessions (if you exit python
 ;; with C-d in py-shell) and defines the command `ipython-to-doctest', which
 ;; can be used to convert bits of a ipython session into something that can be
 ;; used for doctests. To install, put this file somewhere in your emacs
 ;; `load-path' [1] and add the following line to your ~/.emacs file (the first
 ;; line only needed if the default (``"ipython"``) is wrong)::
 ;;
 ;;   (setq ipython-command "/SOME-PATH/ipython")
 ;;   (require 'ipython)
 ;;
 ;; Ipython will be set as the default python shell, but only if the ipython
 ;; executable is in the path. For ipython sessions autocompletion with <tab>
 ;; is also enabled (experimental feature!). Please also note that all the
 ;; terminal functions in py-shell are handled by emacs's comint, **not** by
 ;; (i)python, so importing readline etc. will have 0 effect.
 ;;
 ;; To start an interactive ipython session run `py-shell' with ``M-x py-shell``
 ;; (or the default keybinding ``C-c C-!``).
 ;;
+;; You can customize the arguments passed to the IPython instance at startup by
+;; setting the ``py-python-command-args`` variable.  For example, to start
+;; always in ``pylab`` mode with hardcoded light-background colors, you can
+;; use::
+;;
+;; (setq py-python-command-args '("-pylab" "-colors" "LightBG"))
+;;
+;;
 ;; NOTE: This mode is currently somewhat alpha and although I hope that it
 ;; will work fine for most cases, doing certain things (like the
 ;; autocompletion and a decent scheme to switch between python interpreters)
 ;; properly will also require changes to ipython that will likely have to wait
 ;; for a larger rewrite scheduled some time in the future.
-;; 
-;; Also note that you currently NEED THE CVS VERSION OF PYTHON.EL.
+;;
 ;;
 ;; Further note that I don't know whether this runs under windows or not and
 ;; that if it doesn't I can't really help much, not being afflicted myself.
 ;;
 ;;
 ;; Hints for effective usage
 ;; -------------------------
 ;;
-;; - IMO the best feature by far of the ipython/emacs combo is how much easier it
-;;   makes it to find and fix bugs thanks to the ``%pdb on``/ pdbtrack combo. Try
-;;   it: first in the ipython to shell do ``%pdb on`` then do something that will
-;;   raise an exception (FIXME nice example) -- and be amazed how easy it is to
-;;   inspect the live objects in each stack frames and to jump to the
-;;   corresponding sourcecode locations as you walk up and down the stack trace
-;;   (even without ``%pdb on`` you can always use ``C-c -`` (`py-up-exception')
-;;   to jump to the corresponding source code locations).
+;; - IMO the best feature by far of the ipython/emacs combo is how much easier
+;;   it makes it to find and fix bugs thanks to the ``%pdb on or %debug``/
+;;   pdbtrack combo. Try it: first in the ipython to shell do ``%pdb on`` then
+;;   do something that will raise an exception (FIXME nice example), or type
+;;   ``%debug`` after the exception has been raised.  YOu'll be amazed at how
+;;   easy it is to inspect the live objects in each stack frames and to jump to
+;;   the corresponding sourcecode locations as you walk up and down the stack
+;;   trace (even without ``%pdb on`` you can always use ``C-c -``
+;;   (`py-up-exception') to jump to the corresponding source code locations).
 ;;
 ;; - emacs gives you much more powerful commandline editing and output searching
 ;;   capabilities than ipython-standalone -- isearch is your friend if you
 ;;   quickly want to print 'DEBUG ...' to stdout out etc.
 ;;
 ;; - This is not really specific to ipython, but for more convenient history
 ;;   access you might want to add something like the following to *the beggining*
 ;;   of your ``.emacs`` (if you want behavior that's more similar to stand-alone
 ;;   ipython, you can change ``meta p`` etc. for ``control p``)::
 ;;
 ;;         (require 'comint)
-;;         (define-key comint-mode-map [(meta p)] 
+;;         (define-key comint-mode-map [(meta p)]
 ;;           'comint-previous-matching-input-from-input)
-;;         (define-key comint-mode-map [(meta n)] 
+;;         (define-key comint-mode-map [(meta n)]
 ;;           'comint-next-matching-input-from-input)
 ;;         (define-key comint-mode-map [(control meta n)]
 ;;            'comint-next-input)
 ;;         (define-key comint-mode-map [(control meta p)]
 ;;            'comint-previous-input)
 ;;
 ;; - Be aware that if you customize py-python-command previously, this value
 ;;   will override what ipython.el does (because loading the customization
 ;;   variables comes later).
 ;;
 ;; Please send comments and feedback to the ipython-list
-;; (<[e-mail]>) where I (a.s.) or someone else will try to
-;; answer them (it helps if you specify your emacs version, OS etc; 
+;; (<[e-mail]>) where I (a.s.) or someone else will try to
+;; answer them (it helps if you specify your emacs version, OS etc;
 ;; familiarity with <http://www.catb.org/~esr/faqs/smart-questions.html> might
 ;; speed up things further).
 ;;
 ;; Footnotes:
 ;;
 ;;     [1] If you don't know what `load-path' is, C-h v load-path will tell
 ;;     you; if required you can also add a new directory. So assuming that
 ;;     ipython.el resides in ~/el/, put this in your emacs:
 ;;
 ;;
 ;;           (add-to-list 'load-path "~/el")
 ;;           (setq ipython-command "/some-path/ipython")
 ;;           (require 'ipython)
 ;;
 ;;
 ;;
 ;;
 ;; TODO:
 ;;      - do autocompletion properly
 ;;      - implement a proper switching between python interpreters
 ;;
 ;; BUGS:
 ;;      - neither::
 ;;
 ;;         (py-shell "-c print 'FOOBAR'")
-;;       
+;;
 ;;        nor::
-;;       
-;;         (let ((py-python-command-args (append py-python-command-args 
+;;
+;;         (let ((py-python-command-args (append py-python-command-args
 ;;                                              '("-c" "print 'FOOBAR'"))))
 ;;           (py-shell))
 ;;
 ;;        seem to print anything as they should
 ;;
 ;;      - look into init priority issues with `py-python-command' (if it's set
 ;;        via custom)
 
 
 ;;; Code
 (require 'cl)
 (require 'shell)
 (require 'executable)
 (require 'ansi-color)
 
 (defcustom ipython-command "ipython"
   "*Shell command used to start ipython."
-  :type 'string 
+  :type 'string
   :group 'python)
 
 ;; Users can set this to nil
 (defvar py-shell-initial-switch-buffers t
   "If nil, don't switch to the *Python* buffer on the first call to
   `py-shell'.")
 
 (defvar ipython-backup-of-py-python-command nil
   "HACK")
 
-  
+
 (defvar ipython-de-input-prompt-regexp "\\(?:
 In \\[[0-9]+\\]: *.*
 ----+> \\(.*
 \\)[\n]?\\)\\|\\(?:
 In \\[[0-9]+\\]: *\\(.*
 \\)\\)\\|^[ ]\\{3\\}[.]\\{3,\\}: *\\(.*
 \\)"
   "A regular expression to match the IPython input prompt and the python
 command after it. The first match group is for a command that is rewritten,
 the second for a 'normal' command, and the third for a multiline command.")
 (defvar ipython-de-output-prompt-regexp "^Out\\[[0-9]+\\]: "
   "A regular expression to match the output prompt of IPython.")
 
 
 (if (not (executable-find ipython-command))
     (message (format "Can't find executable %s - ipython.el *NOT* activated!!!"
                      ipython-command))
     ;; XXX load python-mode, so that we can screw around with its variables
     ;; this has the disadvantage that python-mode is loaded even if no
     ;; python-file is ever edited etc. but it means that `py-shell' works
     ;; without loading a python-file first. Obviously screwing around with
     ;; python-mode's variables like this is a mess, but well.
     (require 'python-mode)
     ;; turn on ansi colors for ipython and activate completion
     (defun ipython-shell-hook ()
       ;; the following is to synchronize dir-changes
       (make-local-variable 'shell-dirstack)
       (setq shell-dirstack nil)
       (make-local-variable 'shell-last-dir)
       (setq shell-last-dir nil)
       (make-local-variable 'shell-dirtrackp)
       (setq shell-dirtrackp t)
       (add-hook 'comint-input-filter-functions 'shell-directory-tracker nil t)
 
       (ansi-color-for-comint-mode-on)
       (define-key py-shell-map [tab] 'ipython-complete)
       ;; Add this so that tab-completion works both in X11 frames and inside
       ;; terminals (such as when emacs is called with -nw).
       (define-key py-shell-map "\t" 'ipython-complete)
       ;;XXX this is really just a cheap hack, it only completes symbols in the
       ;;interactive session -- useful nonetheless.
       (define-key py-mode-map [(meta tab)] 'ipython-complete)
-      
+
       )
     (add-hook 'py-shell-hook 'ipython-shell-hook)
     ;; Regular expression that describes tracebacks for IPython in context and
-    ;; verbose mode. 
-  
+    ;; verbose mode.
+
     ;;Adapt python-mode settings for ipython.
     ;; (this works for %xmode 'verbose' or 'context')
 
-    ;; XXX putative regexps for syntax errors; unfortunately the 
+    ;; XXX putative regexps for syntax errors; unfortunately the
     ;;     current python-mode traceback-line-re scheme is too primitive,
     ;;     so it's either matching syntax errors, *or* everything else
     ;;     (XXX: should ask Fernando for a change)
     ;;"^   File \"\\(.*?\\)\", line \\([0-9]+\\).*\n.*\n.*\nSyntaxError:"
     ;;^   File \"\\(.*?\\)\", line \\([0-9]+\\)"
 
     (setq py-traceback-line-re
           "\\(^[^\t >].+?\\.py\\).*\n   +[0-9]+[^\00]*?\n-+> \\([0-9]+\\)+")
 
-    
+
     ;; Recognize the ipython pdb, whose prompt is 'ipdb>' or  'ipydb>'
     ;;instead of '(Pdb)'
     (setq py-pdbtrack-input-prompt "\n[(<]*[Ii]?[Pp]y?db[>)]+ ")
     (setq pydb-pydbtrack-input-prompt "\n[(]*ipydb[>)]+ ")
-    
+
     (setq py-shell-input-prompt-1-regexp "^In \\[[0-9]+\\]: *"
           py-shell-input-prompt-2-regexp "^   [.][.][.]+: *" )
     ;; select a suitable color-scheme
     (unless (member "-colors" py-python-command-args)
-      (setq py-python-command-args 
-            (nconc py-python-command-args 
+      (setq py-python-command-args
+            (nconc py-python-command-args
                    (list "-colors"
-                         (cond  
+                         (cond
                            ((eq frame-background-mode 'dark)
                             "Linux")
                            ((eq frame-background-mode 'light)
                             "LightBG")
                            (t ; default (backg-mode isn't always set by XEmacs)
                             "LightBG"))))))
     (unless (equal ipython-backup-of-py-python-command py-python-command)
       (setq ipython-backup-of-py-python-command py-python-command))
     (setq py-python-command ipython-command))
 
 
 ;; MODIFY py-shell so that it loads the editing history
 (defadvice py-shell (around py-shell-with-history)
   "Add persistent command-history support (in
 $PYTHONHISTORY (or \"~/.ipython/history\", if we use IPython)). Also, if
 `py-shell-initial-switch-buffers' is nil, it only switches to *Python* if that
 buffer already exists."
   (if (comint-check-proc "*Python*")
       ad-do-it
     (setq comint-input-ring-file-name
           (if (string-equal py-python-command ipython-command)
               (concat (or (getenv "IPYTHONDIR") "~/.ipython") "/history")
             (or (getenv "PYTHONHISTORY") "~/.python-history.py")))
     (comint-read-input-ring t)
     (let ((buf (current-buffer)))
       ad-do-it
       (unless py-shell-initial-switch-buffers
         (switch-to-buffer-other-window buf)))))
 (ad-activate 'py-shell)
 ;; (defadvice py-execute-region (before py-execute-buffer-ensure-process)
 ;;   "HACK: test that ipython is already running before executing something.
 ;;   Doing this properly seems not worth the bother (unless people actually
 ;;   request it)."
 ;; (unless (comint-check-proc "*Python*")
 ;;     (error "Sorry you have to first do M-x py-shell to send something to ipython.")))
 ;; (ad-activate 'py-execute-region)
 
 (defadvice py-execute-region (around py-execute-buffer-ensure-process)
   "HACK: if `py-shell' is not active or ASYNC is explicitly desired, fall back
-  to python instead of ipython." 
+  to python instead of ipython."
   (let ((py-which-shell (if (and (comint-check-proc "*Python*") (not async))
 			    py-python-command
 			  ipython-backup-of-py-python-command)))
     ad-do-it))
 (ad-activate 'py-execute-region)
 
 (defun ipython-to-doctest (start end)
   "Transform a cut-and-pasted bit from an IPython session into something that
 looks like it came from a normal interactive python session, so that it can
 be used in doctests. Example:
 
 
     In [1]: import sys
-    
+
     In [2]: sys.stdout.write 'Hi!\n'
     ------> sys.stdout.write ('Hi!\n')
     Hi!
-    
+
     In [3]: 3 + 4
     Out[3]: 7
-    
+
 gets converted to:
 
     >>> import sys
     >>> sys.stdout.write ('Hi!\n')
     Hi!
     >>> 3 + 4
     7
 
 "
   (interactive "*r\n")
   ;(message (format "###DEBUG s:%de:%d" start end))
   (save-excursion
     (save-match-data
-      ;; replace ``In [3]: bla`` with ``>>> bla`` and 
+      ;; replace ``In [3]: bla`` with ``>>> bla`` and
       ;;         ``... :   bla`` with ``...    bla``
       (goto-char start)
       (while (re-search-forward ipython-de-input-prompt-regexp end t)
         ;(message "finding 1")
         (cond ((match-string 3)         ;continued
                (replace-match "... \\3" t nil))
               (t
                (replace-match ">>> \\1\\2" t nil))))
       ;; replace ``
       (goto-char start)
       (while (re-search-forward ipython-de-output-prompt-regexp end t)
         (replace-match "" t nil)))))
 
-(defvar ipython-completion-command-string 
+(defvar ipython-completion-command-string
   "print ';'.join(__IP.Completer.all_completions('%s')) #PYTHON-MODE SILENT\n"
   "The string send to ipython to query for all possible completions")
 
 
 ;; xemacs doesn't have `comint-preoutput-filter-functions' so we'll try the
 ;; following wonderful hack to work around this case
 (if (featurep 'xemacs)
     ;;xemacs
     (defun ipython-complete ()
       "Try to complete the python symbol before point. Only knows about the stuff
 in the current *Python* session."
       (interactive)
       (let* ((ugly-return nil)
              (sep ";")
              (python-process (or (get-buffer-process (current-buffer))
                                  ;XXX hack for .py buffers
                                  (get-process py-which-bufname)))
              ;; XXX currently we go backwards to find the beginning of an
              ;; expression part; a more powerful approach in the future might be
              ;; to let ipython have the complete line, so that context can be used
              ;; to do things like filename completion etc.
              (beg (save-excursion (skip-chars-backward "a-z0-9A-Z_." (point-at-bol))
                                   (point)))
              (end (point))
              (pattern (buffer-substring-no-properties beg end))
              (completions nil)
              (completion-table nil)
              completion
              (comint-output-filter-functions
-              (append comint-output-filter-functions 
+              (append comint-output-filter-functions
                       '(ansi-color-filter-apply
-                        (lambda (string) 
+                        (lambda (string)
                                         ;(message (format "DEBUG filtering: %s" string))
                           (setq ugly-return (concat ugly-return string))
-                          (delete-region comint-last-output-start 
+                          (delete-region comint-last-output-start
                                          (process-mark (get-buffer-process (current-buffer)))))))))
                                         ;(message (format "#DEBUG pattern: '%s'" pattern))
-        (process-send-string python-process 
+        (process-send-string python-process
                               (format ipython-completion-command-string pattern))
         (accept-process-output python-process)
                                         ;(message (format "DEBUG return: %s" ugly-return))
-        (setq completions 
+        (setq completions
               (split-string (substring ugly-return 0 (position ?\n ugly-return)) sep))
         (setq completion-table (loop for str in completions
                                      collect (list str nil)))
         (setq completion (try-completion pattern completion-table))
         (cond ((eq completion t))
               ((null completion)
                (message "Can't find completion for \"%s\"" pattern)
                (ding))
               ((not (string= pattern completion))
                (delete-region beg end)
                (insert completion))
               (t
                (message "Making completion list...")
                (with-output-to-temp-buffer "*Python Completions*"
                  (display-completion-list (all-completions pattern completion-table)))
                (message "Making completion list...%s" "done")))))
   ;; emacs
   (defun ipython-complete ()
     "Try to complete the python symbol before point. Only knows about the stuff
 in the current *Python* session."
     (interactive)
     (let* ((ugly-return nil)
            (sep ";")
            (python-process (or (get-buffer-process (current-buffer))
                                         ;XXX hack for .py buffers
                                (get-process py-which-bufname)))
            ;; XXX currently we go backwards to find the beginning of an
            ;; expression part; a more powerful approach in the future might be
            ;; to let ipython have the complete line, so that context can be used
            ;; to do things like filename completion etc.
            (beg (save-excursion (skip-chars-backward "a-z0-9A-Z_./" (point-at-bol))
-                                (point))) 
+                                (point)))
            (end (point))
            (pattern (buffer-substring-no-properties beg end))
            (completions nil)
            (completion-table nil)
            completion
          (comint-preoutput-filter-functions
-          (append comint-preoutput-filter-functions 
+          (append comint-preoutput-filter-functions
                   '(ansi-color-filter-apply
-                    (lambda (string) 
+                    (lambda (string)
                       (setq ugly-return (concat ugly-return string))
                       "")))))
-      (process-send-string python-process 
+      (process-send-string python-process
                             (format ipython-completion-command-string pattern))
       (accept-process-output python-process)
-      (setq completions 
+      (setq completions
             (split-string (substring ugly-return 0 (position ?\n ugly-return)) sep))
                                         ;(message (format "DEBUG completions: %S" completions))
       (setq completion-table (loop for str in completions
                                    collect (list str nil)))
       (setq completion (try-completion pattern completion-table))
       (cond ((eq completion t))
             ((null completion)
              (message "Can't find completion for \"%s\"" pattern)
              (ding))
             ((not (string= pattern completion))
              (delete-region beg end)
              (insert completion))
             (t
              (message "Making completion list...")
              (with-output-to-temp-buffer "*IPython Completions*"
                (display-completion-list (all-completions pattern completion-table)))
              (message "Making completion list...%s" "done")))))
 )
 
 ;;; autoindent support: patch sent in by Jin Liu <[e-mail]>,
 ;;; originally written by [e-mail]
 ;;; Minor modifications by fperez for xemacs compatibility.
 
 (defvar ipython-autoindent t
  "If non-nil, enable autoindent for IPython shell through python-mode.")
 
 (defvar ipython-indenting-buffer-name "*IPython Indentation Calculation*"
  "Temporary buffer for indenting multiline statement.")
 
 (defun ipython-get-indenting-buffer ()
  "Return a temporary buffer set in python-mode. Create one if necessary."
  (let ((buf (get-buffer-create ipython-indenting-buffer-name)))
    (set-buffer buf)
    (unless (eq major-mode 'python-mode)
      (python-mode))
    buf))
 
 (defvar ipython-indentation-string nil
  "Indentation for the next line in a multiline statement.")
 
 (defun ipython-send-and-indent ()
  "Send the current line to IPython, and calculate the indentation for
 the next line."
  (interactive)
  (if ipython-autoindent
      (let ((line (buffer-substring (point-at-bol) (point)))
            (after-prompt1)
            (after-prompt2))
        (save-excursion
            (comint-bol t)
            (if (looking-at py-shell-input-prompt-1-regexp)
                (setq after-prompt1 t)
              (setq after-prompt2 (looking-at py-shell-input-prompt-2-regexp)))
            (with-current-buffer (ipython-get-indenting-buffer)
              (when after-prompt1
                (erase-buffer))
              (when (or after-prompt1 after-prompt2)
                (delete-region (point-at-bol) (point))
                (insert line)
                (newline-and-indent))))))
  ;; send input line to ipython interpreter
  (comint-send-input))
 
 (defun ipython-indentation-hook (string)
  "Insert indentation string if py-shell-input-prompt-2-regexp
 matches last process output."
  (let* ((start-marker (or comint-last-output-start
                           (point-min-marker)))
         (end-marker (process-mark (get-buffer-process (current-buffer))))
         (text (ansi-color-filter-apply (buffer-substring start-marker end-marker))))
    ;; XXX if `text' matches both pattern, it MUST be the last prompt-2
    (when (and (string-match py-shell-input-prompt-2-regexp text)
 	      (not (string-match "\n$" text)))
      (with-current-buffer (ipython-get-indenting-buffer)
        (setq ipython-indentation-string
 	     (buffer-substring (point-at-bol) (point))))
      (goto-char end-marker)
      (insert ipython-indentation-string)
      (setq ipython-indentation-string nil))))
 
 (add-hook 'py-shell-hook
          (lambda ()
            (add-hook 'comint-output-filter-functions
                      'ipython-indentation-hook)))
 
 (define-key py-shell-map (kbd "RET") 'ipython-send-and-indent)
 ;;; / end autoindent support
 
 (provide 'ipython)