Merge with upstream

IPython/kernel/core/tests/test_interpreter.py

@@ -0,0 +1,26 b''
+# encoding: utf-8
+
+"""This file contains unittests for the interpreter.py module."""
+
+__docformat__ = "restructuredtext en"
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008  The IPython Development Team                           
+#                                                                             
+#  Distributed under the terms of the BSD License.  The full license is in    
+#  the file COPYING, distributed as part of this software.                    
+#-----------------------------------------------------------------------------
+                                                                              
+#-----------------------------------------------------------------------------
+# Imports                                                                     
+#-----------------------------------------------------------------------------
+
+from IPython.kernel.core.interpreter import Interpreter
+
+def test_unicode():
+    """ Test unicode handling with the interpreter.
+    """
+    i = Interpreter()
+    i.execute_python(u'print "ù"')
+    i.execute_python('print "ù"')
+

IPython/tests/test_genutils.py

@@ -0,0 +1,32 b''
+# encoding: utf-8
+
+"""Tests for genutils.py"""
+
+__docformat__ = "restructuredtext en"
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+from IPython import genutils
+
+
+def test_get_home_dir():
+    """Make sure we can get the home directory."""
+    home_dir = genutils.get_home_dir()
+
+def test_get_ipython_dir():
+    """Make sure we can get the ipython directory."""
+    ipdir = genutils.get_ipython_dir()
+
+def test_get_security_dir():
+    """Make sure we can get the ipython/security directory."""
+    sdir = genutils.get_security_dir()
+    
  No newline at end of file

IPython/UserConfig/ipythonrc-physics

@@ -1,43 +1,45 b''
 # -*- Mode: Shell-Script -*-  Not really, but shows comments correctly
 #***************************************************************************
 #
 # Configuration file for ipython -- ipythonrc format
 #
 # The format of this file is one of 'key value' lines.
 # Lines containing only whitespace at the beginning and then a # are ignored
 # as comments. But comments can NOT be put on lines with data.
 #***************************************************************************
 
 # If this file is found in the user's ~/.ipython directory as
 # ipythonrc-physics, it can be loaded by calling passing the '-profile
 # physics' (or '-p physics') option to IPython.
 
 # This profile loads modules useful for doing interactive calculations with
 # physical quantities (with units). It relies on modules from Konrad Hinsen's
 # ScientificPython (http://dirac.cnrs-orleans.fr/ScientificPython/)
 
 # First load basic user configuration
 include ipythonrc
 
 # import ...
 # Module with alternate input syntax for PhysicalQuantity objects.
 import_mod IPython.Extensions.PhysicalQInput
 
 # from ... import *
 # math CANNOT be imported after PhysicalQInteractive. It will override the
 # functions defined there.
 import_all math IPython.Extensions.PhysicalQInteractive
 
 # from ... import ...
 import_some  
 
 # code
 execute q = PhysicalQuantityInteractive
 execute g = PhysicalQuantityInteractive('9.8 m/s**2')
 ececute rad = pi/180.
 execute print '*** q is an alias for PhysicalQuantityInteractive'
 execute print '*** g = 9.8 m/s^2 has been defined'
 execute print '*** rad = pi/180  has been defined'
+execute import ipy_constants as C
+execute print '*** C is the physical constants module'
 
 # Files to execute
 execfile 

IPython/config/api.py

@@ -1,104 +1,106 b''
 # encoding: utf-8
 
 """This is the official entry point to IPython's configuration system.  """
 
 __docformat__ = "restructuredtext en"
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
 
 import os
-from IPython.config.cutils import get_home_dir, get_ipython_dir
+from os.path import join as pjoin
+
+from IPython.genutils import get_home_dir, get_ipython_dir
 from IPython.external.configobj import ConfigObj
 
 # Traitlets config imports
 from IPython.config import traitlets
 from IPython.config.config import *
 from traitlets import *
 
 class ConfigObjManager(object):
     
     def __init__(self, configObj, filename):
         self.current = configObj
         self.current.indent_type = '    '
         self.filename = filename
         # self.write_default_config_file()
         
     def get_config_obj(self):
         return self.current
     
     def update_config_obj(self, newConfig):
         self.current.merge(newConfig)
         
     def update_config_obj_from_file(self, filename):
         newConfig = ConfigObj(filename, file_error=False)
         self.current.merge(newConfig)
         
     def update_config_obj_from_default_file(self, ipythondir=None):
         fname = self.resolve_file_path(self.filename, ipythondir)
         self.update_config_obj_from_file(fname)
 
     def write_config_obj_to_file(self, filename):
         f = open(filename, 'w')
         self.current.write(f)
         f.close()
 
     def write_default_config_file(self):
         ipdir = get_ipython_dir()
-        fname = ipdir + '/' + self.filename
+        fname = pjoin(ipdir, self.filename)
         if not os.path.isfile(fname):
             print "Writing the configuration file to: " + fname
             self.write_config_obj_to_file(fname)
     
     def _import(self, key):
         package = '.'.join(key.split('.')[0:-1])
         obj = key.split('.')[-1]
         execString = 'from %s import %s' % (package, obj)
         exec execString
         exec 'temp = %s' % obj 
         return temp
 
     def resolve_file_path(self, filename, ipythondir = None):
         """Resolve filenames into absolute paths.
 
         This function looks in the following directories in order:
 
         1.  In the current working directory or by absolute path with ~ expanded
         2.  In ipythondir if that is set
         3.  In the IPYTHONDIR environment variable if it exists
         4.  In the ~/.ipython directory
 
         Note: The IPYTHONDIR is also used by the trunk version of IPython so
                changing it will also affect it was well.
         """
 
         # In cwd or by absolute path with ~ expanded
         trythis = os.path.expanduser(filename)
         if os.path.isfile(trythis):
             return trythis
 
         # In ipythondir if it is set
         if ipythondir is not None:
-            trythis = ipythondir + '/' + filename
+            trythis = pjoin(ipythondir, filename)
             if os.path.isfile(trythis):
                 return trythis        
 
-        trythis = get_ipython_dir() + '/' + filename
+        trythis = pjoin(get_ipython_dir(), filename)
         if os.path.isfile(trythis):
             return trythis
 
         return None
 
 
     
     
 
     

IPython/config/cutils.py

@@ -1,99 +1,34 b''
 # encoding: utf-8
 
 """Configuration-related utilities for all IPython."""
 
 __docformat__ = "restructuredtext en"
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
 
 import os
 import sys
 
 #---------------------------------------------------------------------------
 # Normal code begins
 #---------------------------------------------------------------------------
 
-class HomeDirError(Exception):
-    pass
-
-def get_home_dir():
-    """Return the closest possible equivalent to a 'home' directory.
-
-    We first try $HOME.  Absent that, on NT it's $HOMEDRIVE\$HOMEPATH.
-
-    Currently only Posix and NT are implemented, a HomeDirError exception is
-    raised for all other OSes. """
-
-    isdir = os.path.isdir
-    env = os.environ
-    try:
-        homedir = env['HOME']
-        if not isdir(homedir):
-            # in case a user stuck some string which does NOT resolve to a
-            # valid path, it's as good as if we hadn't foud it
-            raise KeyError
-        return homedir
-    except KeyError:
-        if os.name == 'posix':
-            raise HomeDirError,'undefined $HOME, IPython can not proceed.'
-        elif os.name == 'nt':
-            # For some strange reason, win9x returns 'nt' for os.name.
-            try:
-                homedir = os.path.join(env['HOMEDRIVE'],env['HOMEPATH'])
-                if not isdir(homedir):
-                    homedir = os.path.join(env['USERPROFILE'])
-                    if not isdir(homedir):
-                        raise HomeDirError
-                return homedir
-            except:
-                try:
-                    # Use the registry to get the 'My Documents' folder.
-                    import _winreg as wreg
-                    key = wreg.OpenKey(wreg.HKEY_CURRENT_USER,
-                                       "Software\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders")
-                    homedir = wreg.QueryValueEx(key,'Personal')[0]
-                    key.Close()
-                    if not isdir(homedir):
-                        e = ('Invalid "Personal" folder registry key '
-                             'typically "My Documents".\n'
-                             'Value: %s\n'
-                             'This is not a valid directory on your system.' %
-                             homedir)
-                        raise HomeDirError(e)
-                    return homedir
-                except HomeDirError:
-                    raise
-                except:
-                    return 'C:\\'
-        elif os.name == 'dos':
-            # Desperate, may do absurd things in classic MacOS. May work under DOS.
-            return 'C:\\'
-        else:
-            raise HomeDirError,'support for your operating system not implemented.'
-
-def get_ipython_dir():
-    ipdir_def = '.ipython'
-    home_dir = get_home_dir()
-    ipdir = os.path.abspath(os.environ.get('IPYTHONDIR',
-                                           os.path.join(home_dir,ipdir_def)))
-    return ipdir
-
 def import_item(key):
     """
     Import and return bar given the string foo.bar.
     """
     package = '.'.join(key.split('.')[0:-1])
     obj = key.split('.')[-1]
     execString = 'from %s import %s' % (package, obj)
     exec execString
     exec 'temp = %s' % obj 
     return temp

IPython/frontend/_process/killableprocess.py

@@ -1,168 +1,179 b''
 # Addapted from killableprocess.py.  
 #______________________________________________________________________________
 #
 # killableprocess - subprocesses which can be reliably killed
 #
 # Parts of this module are copied from the subprocess.py file contained
 # in the Python distribution.
 #
 # Copyright (c) 2003-2004 by Peter Astrand <[e-mail]>
 #
 # Additions and modifications written by Benjamin Smedberg
 # <[e-mail]> are Copyright (c) 2006 by the Mozilla Foundation
 # <http://www.mozilla.org/>
 #
 # By obtaining, using, and/or copying this software and/or its
 # associated documentation, you agree that you have read, understood,
 # and will comply with the following terms and conditions:
 #
 # Permission to use, copy, modify, and distribute this software and
 # its associated documentation for any purpose and without fee is
 # hereby granted, provided that the above copyright notice appears in
 # all copies, and that both that copyright notice and this permission
 # notice appear in supporting documentation, and that the name of the
 # author not be used in advertising or publicity pertaining to
 # distribution of the software without specific, written prior
 # permission.
 #
 # THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
 # INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS.
 # IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, INDIRECT OR
 # CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
 # OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
 # NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
 # WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 
 r"""killableprocess - Subprocesses which can be reliably killed
 
 This module is a subclass of the builtin "subprocess" module. It allows
 processes that launch subprocesses to be reliably killed on Windows (via the Popen.kill() method.
 
 It also adds a timeout argument to Wait() for a limited period of time before
 forcefully killing the process.
 
 Note: On Windows, this module requires Windows 2000 or higher (no support for
 Windows 95, 98, or NT 4.0). It also requires ctypes, which is bundled with
 Python 2.5+ or available from http://python.net/crew/theller/ctypes/
 """
 
 import subprocess
 from subprocess import PIPE
 import sys
 import os
-import time
 import types
 
 try:
     from subprocess import CalledProcessError
 except ImportError:
     # Python 2.4 doesn't implement CalledProcessError
     class CalledProcessError(Exception):
         """This exception is raised when a process run by check_call() returns
         a non-zero exit status. The exit status will be stored in the
         returncode attribute."""
         def __init__(self, returncode, cmd):
             self.returncode = returncode
             self.cmd = cmd
         def __str__(self):
             return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
 
 mswindows = (sys.platform == "win32")
 
+skip = False
+
 if mswindows:
-    import winprocess
+    import platform
+    if platform.uname()[3] == '' or platform.uname()[3] > '6.0.6000':
+        # Killable process does not work under vista when starting for
+        # something else than cmd.
+        skip = True
+    else:
+        import winprocess
 else:
     import signal
 
 if not mswindows:
     def DoNothing(*args):
         pass
 
-class Popen(subprocess.Popen):
+
+if skip:
+ Popen = subprocess.Popen
+else:
+ class Popen(subprocess.Popen):
     if not mswindows:
         # Override __init__ to set a preexec_fn
         def __init__(self, *args, **kwargs):
             if len(args) >= 7:
                 raise Exception("Arguments preexec_fn and after must be passed by keyword.")
 
             real_preexec_fn = kwargs.pop("preexec_fn", None)
             def setpgid_preexec_fn():
                 os.setpgid(0, 0)
                 if real_preexec_fn:
                     apply(real_preexec_fn)
 
             kwargs['preexec_fn'] = setpgid_preexec_fn
 
             subprocess.Popen.__init__(self, *args, **kwargs)
 
     if mswindows:
         def _execute_child(self, args, executable, preexec_fn, close_fds,
                            cwd, env, universal_newlines, startupinfo,
                            creationflags, shell,
                            p2cread, p2cwrite,
                            c2pread, c2pwrite,
                            errread, errwrite):
             if not isinstance(args, types.StringTypes):
                 args = subprocess.list2cmdline(args)
 
             if startupinfo is None:
                 startupinfo = winprocess.STARTUPINFO()
             
             if None not in (p2cread, c2pwrite, errwrite):
                 startupinfo.dwFlags |= winprocess.STARTF_USESTDHANDLES
                 
                 startupinfo.hStdInput = int(p2cread)
                 startupinfo.hStdOutput = int(c2pwrite)
                 startupinfo.hStdError = int(errwrite)
             if shell:
                 startupinfo.dwFlags |= winprocess.STARTF_USESHOWWINDOW
                 startupinfo.wShowWindow = winprocess.SW_HIDE
                 comspec = os.environ.get("COMSPEC", "cmd.exe")
                 args = comspec + " /c " + args
 
             # We create a new job for this process, so that we can kill
             # the process and any sub-processes 
             self._job = winprocess.CreateJobObject()
 
             creationflags |= winprocess.CREATE_SUSPENDED
             creationflags |= winprocess.CREATE_UNICODE_ENVIRONMENT
 
             hp, ht, pid, tid = winprocess.CreateProcess(
                 executable, args,
                 None, None, # No special security
                 1, # Must inherit handles!
                 creationflags,
                 winprocess.EnvironmentBlock(env),
                 cwd, startupinfo)
             
             self._child_created = True
             self._handle = hp
             self._thread = ht
             self.pid = pid
 
             winprocess.AssignProcessToJobObject(self._job, hp)
             winprocess.ResumeThread(ht)
 
             if p2cread is not None:
                 p2cread.Close()
             if c2pwrite is not None:
                 c2pwrite.Close()
             if errwrite is not None:
                 errwrite.Close()
 
     def kill(self, group=True):
         """Kill the process. If group=True, all sub-processes will also be killed."""
         if mswindows:
             if group:
                 winprocess.TerminateJobObject(self._job, 127)
             else:
                 winprocess.TerminateProcess(self._handle, 127)
             self.returncode = 127    
         else:
             if group:
                 os.killpg(self.pid, signal.SIGKILL)
             else:
                 os.kill(self.pid, signal.SIGKILL)
             self.returncode = -9
 
 

IPython/frontend/_process/pipedprocess.py

@@ -1,74 +1,74 b''
 # encoding: utf-8
 """
 Object for encapsulating process execution by using callbacks for stdout, 
 stderr and stdin.
 """
 __docformat__ = "restructuredtext en"
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
 from killableprocess import Popen, PIPE
 from threading import Thread
 from time import sleep
 import os
 
 class PipedProcess(Thread):
     """ Class that encapsulates process execution by using callbacks for
         stdout, stderr and stdin, and providing a reliable way of
         killing it.
     """
 
     def __init__(self, command_string, out_callback, 
                         end_callback=None,):
         """ command_string: the command line executed to start the
         process. 
 
         out_callback: the python callable called on stdout/stderr.
 
         end_callback: an optional callable called when the process
         finishes.
 
         These callbacks are called from a different thread as the
         thread from which is started.
         """
         self.command_string = command_string
         self.out_callback = out_callback
         self.end_callback = end_callback
         Thread.__init__(self)
     
 
     def run(self):
         """ Start the process and hook up the callbacks.
         """
         env = os.environ
         env['TERM'] = 'xterm'
-        process = Popen((self.command_string + ' 2>&1', ), shell=True,
+        process = Popen(self.command_string + ' 2>&1', shell=True,
                                 env=env,
                                 universal_newlines=True,
                                 stdout=PIPE, stdin=PIPE, )
         self.process = process
         while True:
             out_char = process.stdout.read(1)
             if out_char == '':
                 if process.poll() is not None:
                     # The process has finished
                     break
                 else:
                     # The process is not giving any interesting
                     # output. No use polling it immediatly.
                     sleep(0.1)
             else:
                 self.out_callback(out_char)
 
         if self.end_callback is not None:
             self.end_callback()
     
 

IPython/frontend/linefrontendbase.py

@@ -1,294 +1,320 b''
 """
 Base front end class for all line-oriented frontends, rather than
 block-oriented.
 
 Currently this focuses on synchronous frontends.
 """
 __docformat__ = "restructuredtext en"
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
 import re
 
 import IPython
 import sys
+import codeop
+import traceback
 
 from frontendbase import FrontEndBase
 from IPython.kernel.core.interpreter import Interpreter
 
 def common_prefix(strings):
     """ Given a list of strings, return the common prefix between all
         these strings.
     """
     ref = strings[0]
     prefix = ''
     for size in range(len(ref)):
         test_prefix = ref[:size+1]
         for string in strings[1:]:
             if not string.startswith(test_prefix):
                 return prefix
         prefix = test_prefix
 
     return prefix
 
 #-------------------------------------------------------------------------------
 # Base class for the line-oriented front ends
 #-------------------------------------------------------------------------------
 class LineFrontEndBase(FrontEndBase):
     """ Concrete implementation of the FrontEndBase class. This is meant
     to be the base class behind all the frontend that are line-oriented,
     rather than block-oriented.
     """
     
     # We need to keep the prompt number, to be able to increment
     # it when there is an exception.
     prompt_number = 1
 
     # We keep a reference to the last result: it helps testing and
     # programatic control of the frontend. 
     last_result = dict(number=0)
 
     # The input buffer being edited
     input_buffer = ''
 
     # Set to true for debug output
     debug = False
 
     # A banner to print at startup
     banner = None
 
     #--------------------------------------------------------------------------
     # FrontEndBase interface
     #--------------------------------------------------------------------------
 
     def __init__(self, shell=None, history=None, banner=None, *args, **kwargs):
         if shell is None:
             shell = Interpreter()
         FrontEndBase.__init__(self, shell=shell, history=history)
 
         if banner is not None:
             self.banner = banner
+
+    def start(self):
+        """ Put the frontend in a state where it is ready for user
+            interaction.
+        """
         if self.banner is not None:
             self.write(self.banner, refresh=False)
 
         self.new_prompt(self.input_prompt_template.substitute(number=1))
 
 
     def complete(self, line):
         """Complete line in engine's user_ns
         
         Parameters
         ----------
         line : string
         
         Result
         ------
         The replacement for the line and the list of possible completions.
         """
         completions = self.shell.complete(line)
         complete_sep =  re.compile('[\s\{\}\[\]\(\)\=]')
         if completions:
             prefix = common_prefix(completions) 
             residual = complete_sep.split(line)[:-1]
             line = line[:-len(residual)] + prefix
         return line, completions 
  
 
     def render_result(self, result):
         """ Frontend-specific rendering of the result of a calculation
         that has been sent to an engine.
         """
         if 'stdout' in result and result['stdout']:
             self.write('\n' + result['stdout'])
         if 'display' in result and result['display']:
             self.write("%s%s\n" % ( 
                             self.output_prompt_template.substitute(
                                     number=result['number']),
                             result['display']['pprint']
                             ) )
        
 
     def render_error(self, failure):
         """ Frontend-specific rendering of error. 
         """
         self.write('\n\n'+str(failure)+'\n\n')
         return failure
 
 
     def is_complete(self, string):
         """ Check if a string forms a complete, executable set of
         commands.
 
         For the line-oriented frontend, multi-line code is not executed
         as soon as it is complete: the users has to enter two line
         returns.
         """
         if string in ('', '\n'):
             # Prefiltering, eg through ipython0, may return an empty
             # string although some operations have been accomplished. We
             # thus want to consider an empty string as a complete
             # statement.
             return True
         elif ( len(self.input_buffer.split('\n'))>2 
                         and not re.findall(r"\n[\t ]*\n[\t ]*$", string)):
             return False
         else:
-            # Add line returns here, to make sure that the statement is
-            # complete.
-            return FrontEndBase.is_complete(self, string.rstrip() + '\n\n')
+            self.capture_output()
+            try:
+                # Add line returns here, to make sure that the statement is
+                # complete.
+                is_complete = codeop.compile_command(string.rstrip() + '\n\n',
+                            "<string>", "exec")
+                self.release_output()
+            except Exception, e:
+                # XXX: Hack: return True so that the
+                # code gets executed and the error captured.
+                is_complete = True
+            return is_complete
 
 
     def write(self, string, refresh=True):
         """ Write some characters to the display.
 
             Subclass should overide this method.
 
             The refresh keyword argument is used in frontends with an
             event loop, to choose whether the write should trigget an UI
             refresh, and thus be syncrhonous, or not.
         """
         print >>sys.__stderr__, string
 
 
     def execute(self, python_string, raw_string=None):
         """ Stores the raw_string in the history, and sends the
         python string to the interpreter.
         """
         if raw_string is None:
             raw_string = python_string
         # Create a false result, in case there is an exception
         self.last_result = dict(number=self.prompt_number)
         try:
             self.history.input_cache[-1] = raw_string.rstrip()
             result = self.shell.execute(python_string)
             self.last_result = result
             self.render_result(result)
         except:
             self.show_traceback()
         finally:
             self.after_execute()
 
     #--------------------------------------------------------------------------
     # LineFrontEndBase interface
     #--------------------------------------------------------------------------
 
     def prefilter_input(self, string):
-        """ Priflter the input to turn it in valid python.
+        """ Prefilter the input to turn it in valid python.
         """
         string = string.replace('\r\n', '\n')
         string = string.replace('\t', 4*' ')
         # Clean the trailing whitespace
         string = '\n'.join(l.rstrip()  for l in string.split('\n'))
         return string
 
 
     def after_execute(self):
         """ All the operations required after an execution to put the
             terminal back in a shape where it is usable.
         """
         self.prompt_number += 1
         self.new_prompt(self.input_prompt_template.substitute(
                             number=(self.last_result['number'] + 1)))
         # Start a new empty history entry
         self._add_history(None, '')
         self.history_cursor = len(self.history.input_cache) - 1
 
 
     def complete_current_input(self):
         """ Do code completion on current line. 
         """
         if self.debug:
             print >>sys.__stdout__, "complete_current_input", 
         line = self.input_buffer
         new_line, completions = self.complete(line)
         if len(completions)>1:
-            self.write_completion(completions)
-        self.input_buffer = new_line
+            self.write_completion(completions, new_line=new_line)
+        elif not line == new_line:
+            self.input_buffer = new_line
         if self.debug:
+            print >>sys.__stdout__, 'line', line
+            print >>sys.__stdout__, 'new_line', new_line
             print >>sys.__stdout__, completions 
 
 
     def get_line_width(self):
         """ Return the width of the line in characters.
         """
         return 80
 
 
-    def write_completion(self, possibilities):
+    def write_completion(self, possibilities, new_line=None):
         """ Write the list of possible completions.
+
+            new_line is the completed input line that should be displayed
+            after the completion are writen. If None, the input_buffer
+            before the completion is used.
         """
-        current_buffer = self.input_buffer
+        if new_line is None:
+            new_line = self.input_buffer
         
         self.write('\n')
         max_len = len(max(possibilities, key=len)) + 1
         
         # Now we check how much symbol we can put on a line...
         chars_per_line = self.get_line_width()
         symbols_per_line = max(1, chars_per_line/max_len)
 
         pos = 1
         buf = []
         for symbol in possibilities:
             if pos < symbols_per_line:
                 buf.append(symbol.ljust(max_len))
                 pos += 1
             else:
                 buf.append(symbol.rstrip() + '\n')
                 pos = 1
         self.write(''.join(buf))
         self.new_prompt(self.input_prompt_template.substitute(
                             number=self.last_result['number'] + 1))
-        self.input_buffer = current_buffer
+        self.input_buffer = new_line
 
 
     def new_prompt(self, prompt):
         """ Prints a prompt and starts a new editing buffer. 
 
             Subclasses should use this method to make sure that the
             terminal is put in a state favorable for a new line
             input.
         """
         self.input_buffer = ''
         self.write(prompt)
 
 
     #--------------------------------------------------------------------------
     # Private API
     #--------------------------------------------------------------------------
  
     def _on_enter(self):
         """ Called when the return key is pressed in a line editing
             buffer.
         """
         current_buffer = self.input_buffer
         cleaned_buffer = self.prefilter_input(current_buffer)
         if self.is_complete(cleaned_buffer):
             self.execute(cleaned_buffer, raw_string=current_buffer)
         else:
             self.input_buffer += self._get_indent_string(
                                                 current_buffer[:-1])
+            if len(current_buffer.split('\n')) == 2:
+                self.input_buffer += '\t\t'
             if current_buffer[:-1].split('\n')[-1].rstrip().endswith(':'):
                 self.input_buffer += '\t'
 
 
     def _get_indent_string(self, string):
         """ Return the string of whitespace that prefixes a line. Used to
         add the right amount of indendation when creating a new line.
         """
         string = string.replace('\t', ' '*4)
         string = string.split('\n')[-1]
         indent_chars = len(string) - len(string.lstrip())
         indent_string = '\t'*(indent_chars // 4) + \
                             ' '*(indent_chars % 4)
 
         return indent_string
  
     

IPython/frontend/prefilterfrontend.py

@@ -1,223 +1,230 b''
 """
 Frontend class that uses IPython0 to prefilter the inputs.
 
 Using the IPython0 mechanism gives us access to the magics.
 
 This is a transitory class, used here to do the transition between
 ipython0 and ipython1. This class is meant to be short-lived as more
 functionnality is abstracted out of ipython0 in reusable functions and
 is added on the interpreter. This class can be a used to guide this
 refactoring.
 """
 __docformat__ = "restructuredtext en"
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
 import sys
 
 from linefrontendbase import LineFrontEndBase, common_prefix
+from frontendbase import FrontEndBase
 
 from IPython.ipmaker import make_IPython
 from IPython.ipapi import IPApi
 from IPython.kernel.core.redirector_output_trap import RedirectorOutputTrap
 
 from IPython.kernel.core.sync_traceback_trap import SyncTracebackTrap
 
 from IPython.genutils import Term
 import pydoc
 import os
+import sys
 
 
 def mk_system_call(system_call_function, command):
     """ given a os.system replacement, and a leading string command,
         returns a function that will execute the command with the given
         argument string.
     """
     def my_system_call(args):
         system_call_function("%s %s" % (command, args))
     return my_system_call
 
 #-------------------------------------------------------------------------------
 # Frontend class using ipython0 to do the prefiltering. 
 #-------------------------------------------------------------------------------
 class PrefilterFrontEnd(LineFrontEndBase):
     """ Class that uses ipython0 to do prefilter the input, do the
     completion and the magics.
 
     The core trick is to use an ipython0 instance to prefilter the
     input, and share the namespace between the interpreter instance used
     to execute the statements and the ipython0 used for code
     completion...
     """
+
+    debug = False
     
     def __init__(self, ipython0=None, *args, **kwargs):
         """ Parameters:
             -----------
 
             ipython0: an optional ipython0 instance to use for command
             prefiltering and completion.
         """
+        LineFrontEndBase.__init__(self, *args, **kwargs)
+        self.shell.output_trap = RedirectorOutputTrap(
+                            out_callback=self.write,
+                            err_callback=self.write,
+                                            )
+        self.shell.traceback_trap = SyncTracebackTrap(
+                        formatters=self.shell.traceback_trap.formatters,
+                            )
+
+        # Start the ipython0 instance:
         self.save_output_hooks()
         if ipython0 is None:
             # Instanciate an IPython0 interpreter to be able to use the
             # prefiltering.
             # XXX: argv=[] is a bit bold.
-            ipython0 = make_IPython(argv=[])
+            ipython0 = make_IPython(argv=[], 
+                                    user_ns=self.shell.user_ns,
+                                    user_global_ns=self.shell.user_global_ns)
         self.ipython0 = ipython0
         # Set the pager:
         self.ipython0.set_hook('show_in_pager', 
                     lambda s, string: self.write("\n" + string))
         self.ipython0.write = self.write
         self._ip = _ip = IPApi(self.ipython0)
         # Make sure the raw system call doesn't get called, as we don't
         # have a stdin accessible.
         self._ip.system = self.system_call
         # XXX: Muck around with magics so that they work better
         # in our environment
         self.ipython0.magic_ls = mk_system_call(self.system_call, 
                                                             'ls -CF')
         # And now clean up the mess created by ipython0
         self.release_output()
+
+
         if not 'banner' in kwargs and self.banner is None:
-            kwargs['banner'] = self.ipython0.BANNER + """
+            self.banner = self.ipython0.BANNER + """
 This is the wx frontend, by Gael Varoquaux. This is EXPERIMENTAL code."""
 
-        LineFrontEndBase.__init__(self, *args, **kwargs)
-        # XXX: Hack: mix the two namespaces
-        self.shell.user_ns.update(self.ipython0.user_ns)
-        self.ipython0.user_ns = self.shell.user_ns 
-        self.shell.user_global_ns.update(self.ipython0.user_global_ns)
-        self.ipython0.user_global_ns = self.shell.user_global_ns
-
-        self.shell.output_trap = RedirectorOutputTrap(
-                            out_callback=self.write,
-                            err_callback=self.write,
-                                            )
-        self.shell.traceback_trap = SyncTracebackTrap(
-                        formatters=self.shell.traceback_trap.formatters,
-                            )
+        self.start()
 
     #--------------------------------------------------------------------------
     # FrontEndBase interface 
     #--------------------------------------------------------------------------
 
     def show_traceback(self):
         """ Use ipython0 to capture the last traceback and display it.
         """
         self.capture_output()
-        self.ipython0.showtraceback()
+        self.ipython0.showtraceback(tb_offset=-1)
         self.release_output()
 
 
     def execute(self, python_string, raw_string=None):
         if self.debug:
             print 'Executing Python code:', repr(python_string)
         self.capture_output()
         LineFrontEndBase.execute(self, python_string,
                                     raw_string=raw_string)
         self.release_output()
 
 
     def save_output_hooks(self):
         """ Store all the output hooks we can think of, to be able to
         restore them. 
         
         We need to do this early, as starting the ipython0 instance will
         screw ouput hooks.
         """
         self.__old_cout_write = Term.cout.write
         self.__old_cerr_write = Term.cerr.write
         self.__old_stdout = sys.stdout
         self.__old_stderr= sys.stderr
         self.__old_help_output = pydoc.help.output
         self.__old_display_hook = sys.displayhook
 
 
     def capture_output(self):
         """ Capture all the output mechanisms we can think of.
         """
         self.save_output_hooks()
         Term.cout.write = self.write
         Term.cerr.write = self.write
         sys.stdout = Term.cout
         sys.stderr = Term.cerr
         pydoc.help.output = self.shell.output_trap.out
 
 
     def release_output(self):
         """ Release all the different captures we have made.
         """
         Term.cout.write = self.__old_cout_write
         Term.cerr.write = self.__old_cerr_write
         sys.stdout = self.__old_stdout
         sys.stderr = self.__old_stderr
         pydoc.help.output = self.__old_help_output 
         sys.displayhook = self.__old_display_hook 
 
 
     def complete(self, line):
+        # FIXME: This should be factored out in the linefrontendbase
+        # method.
         word = line.split('\n')[-1].split(' ')[-1]
         completions = self.ipython0.complete(word)
         # FIXME: The proper sort should be done in the complete method.
         key = lambda x: x.replace('_', '')
         completions.sort(key=key)
         if completions:
             prefix = common_prefix(completions) 
             line = line[:-len(word)] + prefix
         return line, completions 
  
     
     #--------------------------------------------------------------------------
     # LineFrontEndBase interface 
     #--------------------------------------------------------------------------
 
     def prefilter_input(self, input_string):
         """ Using IPython0 to prefilter the commands to turn them
         in executable statements that are valid Python strings.
         """
         input_string = LineFrontEndBase.prefilter_input(self, input_string)
         filtered_lines = []
         # The IPython0 prefilters sometime produce output. We need to
         # capture it.
         self.capture_output()
         self.last_result = dict(number=self.prompt_number)
         try:
             for line in input_string.split('\n'):
                 filtered_lines.append(
                         self.ipython0.prefilter(line, False).rstrip())
         except:
             # XXX: probably not the right thing to do.
             self.ipython0.showsyntaxerror()
             self.after_execute()
         finally:
             self.release_output()
 
         # Clean up the trailing whitespace, to avoid indentation errors
         filtered_string = '\n'.join(filtered_lines)
         return filtered_string
 
 
     #--------------------------------------------------------------------------
     # PrefilterFrontEnd interface 
     #--------------------------------------------------------------------------
  
     def system_call(self, command_string):
         """ Allows for frontend to define their own system call, to be
             able capture output and redirect input.
         """
         return os.system(command_string)
 
 
     def do_exit(self):
         """ Exit the shell, cleanup and save the history.
         """
         self.ipython0.atexit_operations()
 

IPython/frontend/tests/test_frontendbase.py

@@ -1,157 +1,155 b''
 # encoding: utf-8
 
 """This file contains unittests for the frontendbase module."""
 
 __docformat__ = "restructuredtext en"
 
 #---------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team                         
 #                                                                           
 #  Distributed under the terms of the BSD License.  The full license is in  
 #  the file COPYING, distributed as part of this software.                  
 #---------------------------------------------------------------------------
                                                                             
 #---------------------------------------------------------------------------
 # Imports                                                                   
 #---------------------------------------------------------------------------
 
 import unittest
 
 try:
     from IPython.frontend.asyncfrontendbase import AsyncFrontEndBase
     from IPython.frontend import frontendbase 
     from IPython.kernel.engineservice import EngineService
 except ImportError:
     import nose
     raise nose.SkipTest("This test requires zope.interface, Twisted and Foolscap")
 
+from IPython.testing.decorators import skip
+
 class FrontEndCallbackChecker(AsyncFrontEndBase):
     """FrontEndBase subclass for checking callbacks"""
     def __init__(self, engine=None, history=None):
         super(FrontEndCallbackChecker, self).__init__(engine=engine, 
                                                     history=history)
         self.updateCalled = False
         self.renderResultCalled = False
         self.renderErrorCalled = False
    
     def update_cell_prompt(self, result, blockID=None):
         self.updateCalled = True
         return result
    
     def render_result(self, result):
         self.renderResultCalled = True
         return result
     
     
     def render_error(self, failure):
         self.renderErrorCalled = True
         return failure
     
 
 
     
 class TestAsyncFrontendBase(unittest.TestCase):
     def setUp(self):
         """Setup the EngineService and FrontEndBase"""
         
         self.fb = FrontEndCallbackChecker(engine=EngineService())
     
-    
     def test_implements_IFrontEnd(self):
         assert(frontendbase.IFrontEnd.implementedBy(
                                     AsyncFrontEndBase))
     
-    
     def test_is_complete_returns_False_for_incomplete_block(self):
         """"""
         
         block = """def test(a):"""
         
         assert(self.fb.is_complete(block) == False)
     
     def test_is_complete_returns_True_for_complete_block(self):
         """"""
         
         block = """def test(a): pass"""
             
         assert(self.fb.is_complete(block))
         
         block = """a=3"""
         
         assert(self.fb.is_complete(block))
     
-    
     def test_blockID_added_to_result(self):
         block = """3+3"""
         
         d = self.fb.execute(block, blockID='TEST_ID')
         
         d.addCallback(self.checkBlockID, expected='TEST_ID')
     
     def test_blockID_added_to_failure(self):
         block = "raise Exception()"
         
         d = self.fb.execute(block,blockID='TEST_ID')
         d.addErrback(self.checkFailureID, expected='TEST_ID')
     
     def checkBlockID(self, result, expected=""):
         assert(result['blockID'] == expected)
     
     
     def checkFailureID(self, failure, expected=""):
         assert(failure.blockID == expected)
     
     
     def test_callbacks_added_to_execute(self):
         """test that
         update_cell_prompt
         render_result
         
         are added to execute request
         """
         
         d = self.fb.execute("10+10")
         d.addCallback(self.checkCallbacks)
     
-    
     def checkCallbacks(self, result):
         assert(self.fb.updateCalled)
         assert(self.fb.renderResultCalled)
     
-    
+    @skip("This test fails and lead to an unhandled error in a Deferred.")
     def test_error_callback_added_to_execute(self):
         """test that render_error called on execution error"""
         
         d = self.fb.execute("raise Exception()")
         d.addCallback(self.checkRenderError)
     
     def checkRenderError(self, result):
         assert(self.fb.renderErrorCalled)
     
     def test_history_returns_expected_block(self):
         """Make sure history browsing doesn't fail"""
         
         blocks = ["a=1","a=2","a=3"]
         for b in blocks:
             d = self.fb.execute(b)
         
         # d is now the deferred for the last executed block
         d.addCallback(self.historyTests, blocks)
         
     
     def historyTests(self, result, blocks):
         """historyTests"""
         
         assert(len(blocks) >= 3)
         assert(self.fb.get_history_previous("") == blocks[-2])
         assert(self.fb.get_history_previous("") == blocks[-3])
         assert(self.fb.get_history_next() == blocks[-2])
     
     
     def test_history_returns_none_at_startup(self):
         """test_history_returns_none_at_startup"""
         
         assert(self.fb.get_history_previous("")==None)
         assert(self.fb.get_history_next()==None)
     
     

IPython/frontend/tests/test_process.py

@@ -1,67 +1,67 b''
 # encoding: utf-8
 """
 Test process execution and IO redirection.
 """
 
 __docformat__ = "restructuredtext en"
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is
 #  in the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 from cStringIO import StringIO
 from time import sleep
 import sys
 
 from IPython.frontend._process import PipedProcess
 from IPython.testing import decorators as testdec
 
+
 def test_capture_out():
     """ A simple test to see if we can execute a process and get the output.
     """
     s = StringIO()
     p = PipedProcess('echo 1', out_callback=s.write, )
     p.start()
     p.join()
     result = s.getvalue().rstrip()
     assert result == '1'
 
-# FIXME
[e-mail]("This doesn't work under Windows")
+
 def test_io():
     """ Checks that we can send characters on stdin to the process.
     """
     s = StringIO()
     p = PipedProcess(sys.executable + ' -c "a = raw_input(); print a"', 
                             out_callback=s.write, )
     p.start()
     test_string = '12345\n'
     while not hasattr(p, 'process'):
         sleep(0.1)
     p.process.stdin.write(test_string)
     p.join()
     result = s.getvalue()
     assert result == test_string
 
 
 def test_kill():
     """ Check that we can kill a process, and its subprocess.
     """
     s = StringIO()
     p = PipedProcess(sys.executable + ' -c "a = raw_input();"', 
                             out_callback=s.write, )
     p.start()
     while not hasattr(p, 'process'):
         sleep(0.1)
     p.process.kill() 
     assert p.process.poll() is not None
 
 
 if __name__ == '__main__':
     test_capture_out()
     test_io()
     test_kill()
 

IPython/frontend/wx/console_widget.py

@@ -1,428 +1,436 b''
 # encoding: utf-8
 """
 A Wx widget to act as a console and input commands.
 
 This widget deals with prompts and provides an edit buffer
 restricted to after the last prompt.
 """
 
 __docformat__ = "restructuredtext en"
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is
 #  in the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
 
 import wx
 import wx.stc  as  stc
 
 from wx.py import editwindow
+import time
 import sys
 LINESEP = '\n'
 if sys.platform == 'win32':
     LINESEP = '\n\r'
 
 import re
 
 # FIXME: Need to provide an API for non user-generated display on the
 # screen: this should not be editable by the user.
 
 _DEFAULT_SIZE = 10
 if sys.platform == 'darwin':
     _DEFAULT_STYLE = 12
 
 _DEFAULT_STYLE = {
     'stdout'      : 'fore:#0000FF',
     'stderr'      : 'fore:#007f00',
     'trace'       : 'fore:#FF0000',
 
     'default'     : 'size:%d' % _DEFAULT_SIZE,
     'bracegood'   : 'fore:#00AA00,back:#000000,bold',
     'bracebad'    : 'fore:#FF0000,back:#000000,bold',
 
     # properties for the various Python lexer styles
     'comment'       : 'fore:#007F00',
     'number'        : 'fore:#007F7F',
     'string'        : 'fore:#7F007F,italic',
     'char'          : 'fore:#7F007F,italic',
     'keyword'       : 'fore:#00007F,bold',
     'triple'        : 'fore:#7F0000',
     'tripledouble'  : 'fore:#7F0000',
     'class'         : 'fore:#0000FF,bold,underline',
     'def'           : 'fore:#007F7F,bold',
     'operator'      : 'bold'
     }
 
 # new style numbers
 _STDOUT_STYLE = 15
 _STDERR_STYLE = 16
 _TRACE_STYLE  = 17
 
 
 # system colors
 #SYS_COLOUR_BACKGROUND = wx.SystemSettings.GetColour(wx.SYS_COLOUR_BACKGROUND)
 
 #-------------------------------------------------------------------------------
 # The console widget class
 #-------------------------------------------------------------------------------
 class ConsoleWidget(editwindow.EditWindow):
     """ Specialized styled text control view for console-like workflow.
 
         This widget is mainly interested in dealing with the prompt and
         keeping the cursor inside the editing line.
     """
 
     # This is where the title captured from the ANSI escape sequences are
     # stored.
     title = 'Console'
 
     # The buffer being edited.
     def _set_input_buffer(self, string):
         self.SetSelection(self.current_prompt_pos, self.GetLength())
         self.ReplaceSelection(string)
         self.GotoPos(self.GetLength())
 
     def _get_input_buffer(self):
         """ Returns the text in current edit buffer.
         """
         input_buffer = self.GetTextRange(self.current_prompt_pos,
                                                 self.GetLength())
         input_buffer = input_buffer.replace(LINESEP, '\n')
         return input_buffer
 
     input_buffer = property(_get_input_buffer, _set_input_buffer)
 
     style = _DEFAULT_STYLE.copy()
 
     # Translation table from ANSI escape sequences to color. Override
     # this to specify your colors.
     ANSI_STYLES = {'0;30': [0, 'BLACK'],            '0;31': [1, 'RED'],
                    '0;32': [2, 'GREEN'],            '0;33': [3, 'BROWN'],
                    '0;34': [4, 'BLUE'],             '0;35': [5, 'PURPLE'],
                    '0;36': [6, 'CYAN'],             '0;37': [7, 'LIGHT GREY'],
                    '1;30': [8, 'DARK GREY'],        '1;31': [9, 'RED'],
                    '1;32': [10, 'SEA GREEN'],       '1;33': [11, 'YELLOW'],
                    '1;34': [12, 'LIGHT BLUE'],      '1;35':
                                                      [13, 'MEDIUM VIOLET RED'],
                    '1;36': [14, 'LIGHT STEEL BLUE'], '1;37': [15, 'YELLOW']}
 
     # The color of the carret (call _apply_style() after setting)
     carret_color = 'BLACK'
     
+    # Store the last time a refresh was done
+    _last_refresh_time = 0
+
     #--------------------------------------------------------------------------
     # Public API
     #--------------------------------------------------------------------------
     
     def __init__(self, parent, id=wx.ID_ANY, pos=wx.DefaultPosition, 
-                        size=wx.DefaultSize, style=0, ):
+                        size=wx.DefaultSize, style=wx.WANTS_CHARS, ):
         editwindow.EditWindow.__init__(self, parent, id, pos, size, style)
         self._configure_scintilla()
 
         self.Bind(wx.EVT_KEY_DOWN, self._on_key_down)
         self.Bind(wx.EVT_KEY_UP, self._on_key_up)
     
 
     def write(self, text, refresh=True):
         """ Write given text to buffer, while translating the ansi escape
             sequences.
         """
         # XXX: do not put print statements to sys.stdout/sys.stderr in 
         # this method, the print statements will call this method, as 
         # you will end up with an infinit loop
         title = self.title_pat.split(text)
         if len(title)>1:
             self.title = title[-2]
 
         text = self.title_pat.sub('', text)
         segments = self.color_pat.split(text)
         segment = segments.pop(0)
         self.GotoPos(self.GetLength())
         self.StartStyling(self.GetLength(), 0xFF)
         try:
             self.AppendText(segment)
         except UnicodeDecodeError:
             # XXX: Do I really want to skip the exception?
             pass
         
         if segments:
             for ansi_tag, text in zip(segments[::2], segments[1::2]):
                 self.StartStyling(self.GetLength(), 0xFF)
                 try:
                     self.AppendText(text)
                 except UnicodeDecodeError:
                     # XXX: Do I really want to skip the exception?
                     pass
 
                 if ansi_tag not in self.ANSI_STYLES:
                     style = 0
                 else:
                     style = self.ANSI_STYLES[ansi_tag][0]
 
                 self.SetStyling(len(text), style) 
                 
         self.GotoPos(self.GetLength())
         if refresh:
-            # Maybe this is faster than wx.Yield()
-            self.ProcessEvent(wx.PaintEvent())
-            #wx.Yield()
+            current_time = time.time()
+            if current_time - self._last_refresh_time > 0.03:
+                if sys.platform == 'win32':
+                    wx.SafeYield()
+                else:
+                    wx.Yield()
+                #    self.ProcessEvent(wx.PaintEvent())
+                self._last_refresh_time = current_time 
 
    
     def new_prompt(self, prompt):
         """ Prints a prompt at start of line, and move the start of the
             current block there.
 
             The prompt can be given with ascii escape sequences.
         """
         self.write(prompt, refresh=False)
         # now we update our cursor giving end of prompt
         self.current_prompt_pos = self.GetLength()
         self.current_prompt_line = self.GetCurrentLine()
-        wx.Yield()
         self.EnsureCaretVisible()
 
 
     def scroll_to_bottom(self):
         maxrange = self.GetScrollRange(wx.VERTICAL)
         self.ScrollLines(maxrange)
 
 
     def pop_completion(self, possibilities, offset=0):
         """ Pops up an autocompletion menu. Offset is the offset
             in characters of the position at which the menu should
             appear, relativ to the cursor.
         """
         self.AutoCompSetIgnoreCase(False)
         self.AutoCompSetAutoHide(False)
         self.AutoCompSetMaxHeight(len(possibilities))
         self.AutoCompShow(offset, " ".join(possibilities))
 
 
     def get_line_width(self):
         """ Return the width of the line in characters.
         """
         return self.GetSize()[0]/self.GetCharWidth()
 
     #--------------------------------------------------------------------------
     # EditWindow API
     #--------------------------------------------------------------------------
 
     def OnUpdateUI(self, event):
         """ Override the OnUpdateUI of the EditWindow class, to prevent 
             syntax highlighting both for faster redraw, and for more
             consistent look and feel.
         """
 
     #--------------------------------------------------------------------------
     # Private API
     #--------------------------------------------------------------------------
     
     def _apply_style(self):
         """ Applies the colors for the different text elements and the
             carret.
         """
         self.SetCaretForeground(self.carret_color)
 
         #self.StyleClearAll()
         self.StyleSetSpec(stc.STC_STYLE_BRACELIGHT,  
                           "fore:#FF0000,back:#0000FF,bold")
         self.StyleSetSpec(stc.STC_STYLE_BRACEBAD,
                           "fore:#000000,back:#FF0000,bold")
 
         for style in self.ANSI_STYLES.values():
             self.StyleSetSpec(style[0], "bold,fore:%s" % style[1])
 
    
     def _configure_scintilla(self):
         self.SetEOLMode(stc.STC_EOL_LF)
 
         # Ctrl"+" or Ctrl "-" can be used to zoomin/zoomout the text inside 
         # the widget
         self.CmdKeyAssign(ord('+'), stc.STC_SCMOD_CTRL, stc.STC_CMD_ZOOMIN)
         self.CmdKeyAssign(ord('-'), stc.STC_SCMOD_CTRL, stc.STC_CMD_ZOOMOUT)
         # Also allow Ctrl Shift "=" for poor non US keyboard users. 
         self.CmdKeyAssign(ord('='), stc.STC_SCMOD_CTRL|stc.STC_SCMOD_SHIFT, 
                                             stc.STC_CMD_ZOOMIN)
         
         # Keys: we need to clear some of the keys the that don't play
         # well with a console.
         self.CmdKeyClear(ord('D'), stc.STC_SCMOD_CTRL)
         self.CmdKeyClear(ord('L'), stc.STC_SCMOD_CTRL)
         self.CmdKeyClear(ord('T'), stc.STC_SCMOD_CTRL)
         self.CmdKeyClear(ord('A'), stc.STC_SCMOD_CTRL)
 
         self.SetEOLMode(stc.STC_EOL_CRLF)
         self.SetWrapMode(stc.STC_WRAP_CHAR)
         self.SetWrapMode(stc.STC_WRAP_WORD)
         self.SetBufferedDraw(True)
         self.SetUseAntiAliasing(True)
         self.SetLayoutCache(stc.STC_CACHE_PAGE)
         self.SetUndoCollection(False)
         self.SetUseTabs(True)
         self.SetIndent(4)
         self.SetTabWidth(4)
 
         # we don't want scintilla's autocompletion to choose 
         # automaticaly out of a single choice list, as we pop it up
         # automaticaly
         self.AutoCompSetChooseSingle(False)
         self.AutoCompSetMaxHeight(10)
         # XXX: this doesn't seem to have an effect.
         self.AutoCompSetFillUps('\n')
 
         self.SetMargins(3, 3) #text is moved away from border with 3px
         # Suppressing Scintilla margins
         self.SetMarginWidth(0, 0)
         self.SetMarginWidth(1, 0)
         self.SetMarginWidth(2, 0)
 
         self._apply_style()
 
         # Xterm escape sequences
         self.color_pat = re.compile('\x01?\x1b\[(.*?)m\x02?')
         self.title_pat = re.compile('\x1b]0;(.*?)\x07')
 
         #self.SetEdgeMode(stc.STC_EDGE_LINE)
         #self.SetEdgeColumn(80)
 
         # styles
         p = self.style
         self.StyleSetSpec(stc.STC_STYLE_DEFAULT, p['default'])
         self.StyleClearAll()
         self.StyleSetSpec(_STDOUT_STYLE, p['stdout'])
         self.StyleSetSpec(_STDERR_STYLE, p['stderr'])
         self.StyleSetSpec(_TRACE_STYLE, p['trace'])
 
         self.StyleSetSpec(stc.STC_STYLE_BRACELIGHT, p['bracegood'])
         self.StyleSetSpec(stc.STC_STYLE_BRACEBAD, p['bracebad'])
         self.StyleSetSpec(stc.STC_P_COMMENTLINE, p['comment'])
         self.StyleSetSpec(stc.STC_P_NUMBER, p['number'])
         self.StyleSetSpec(stc.STC_P_STRING, p['string'])
         self.StyleSetSpec(stc.STC_P_CHARACTER, p['char'])
         self.StyleSetSpec(stc.STC_P_WORD, p['keyword'])
         self.StyleSetSpec(stc.STC_P_WORD2, p['keyword'])
         self.StyleSetSpec(stc.STC_P_TRIPLE, p['triple'])
         self.StyleSetSpec(stc.STC_P_TRIPLEDOUBLE, p['tripledouble'])
         self.StyleSetSpec(stc.STC_P_CLASSNAME, p['class'])
         self.StyleSetSpec(stc.STC_P_DEFNAME, p['def'])
         self.StyleSetSpec(stc.STC_P_OPERATOR, p['operator'])
         self.StyleSetSpec(stc.STC_P_COMMENTBLOCK, p['comment'])
 
     def _on_key_down(self, event, skip=True):
         """ Key press callback used for correcting behavior for 
             console-like interfaces: the cursor is constraint to be after
             the last prompt.
 
             Return True if event as been catched.
         """
         catched = True
         # Intercept some specific keys.
         if event.KeyCode == ord('L') and event.ControlDown() :
             self.scroll_to_bottom()
         elif event.KeyCode == ord('K') and event.ControlDown() :
             self.input_buffer = ''
         elif event.KeyCode == ord('A') and event.ControlDown() :
             self.GotoPos(self.GetLength())
             self.SetSelectionStart(self.current_prompt_pos)
             self.SetSelectionEnd(self.GetCurrentPos())
             catched = True
         elif event.KeyCode == ord('E') and event.ControlDown() :
             self.GotoPos(self.GetLength())
             catched = True
         elif event.KeyCode == wx.WXK_PAGEUP:
             self.ScrollPages(-1)
         elif event.KeyCode == wx.WXK_PAGEDOWN:
             self.ScrollPages(1)
         elif event.KeyCode == wx.WXK_UP and event.ShiftDown():
             self.ScrollLines(-1)
         elif event.KeyCode == wx.WXK_DOWN and event.ShiftDown():
             self.ScrollLines(1)
         else:
             catched = False
 
         if self.AutoCompActive():
             event.Skip()
         else:
             if event.KeyCode in (13, wx.WXK_NUMPAD_ENTER) and \
                         event.Modifiers in (wx.MOD_NONE, wx.MOD_WIN):
                 catched = True
                 self.CallTipCancel()
                 self.write('\n', refresh=False)
                 # Under windows scintilla seems to be doing funny stuff to the 
                 # line returns here, but the getter for input_buffer filters 
                 # this out.
                 if sys.platform == 'win32':
                     self.input_buffer = self.input_buffer
                 self._on_enter()
 
             elif event.KeyCode == wx.WXK_HOME:
                 if event.Modifiers in (wx.MOD_NONE, wx.MOD_WIN):
                     self.GotoPos(self.current_prompt_pos)
                     catched = True
 
                 elif event.Modifiers == wx.MOD_SHIFT:
                     # FIXME: This behavior is not ideal: if the selection
                     # is already started, it will jump.
                     self.SetSelectionStart(self.current_prompt_pos) 
                     self.SetSelectionEnd(self.GetCurrentPos())
                     catched = True
 
             elif event.KeyCode == wx.WXK_UP:
                 if self.GetCurrentLine() > self.current_prompt_line:
                     if self.GetCurrentLine() == self.current_prompt_line + 1 \
                             and self.GetColumn(self.GetCurrentPos()) < \
                                 self.GetColumn(self.current_prompt_pos):
                         self.GotoPos(self.current_prompt_pos)
                     else:
                         event.Skip()
                 catched = True
 
             elif event.KeyCode in (wx.WXK_LEFT, wx.WXK_BACK):
                 if self.GetCurrentPos() > self.current_prompt_pos:
                     event.Skip()
                 catched = True
 
             if skip and not catched:
                 # Put the cursor back in the edit region
                 if self.GetCurrentPos() < self.current_prompt_pos:
                     self.GotoPos(self.current_prompt_pos)
                 else:
                     event.Skip()
 
         return catched
 
 
     def _on_key_up(self, event, skip=True):
         """ If cursor is outside the editing region, put it back.
         """
         event.Skip()
         if self.GetCurrentPos() < self.current_prompt_pos:
             self.GotoPos(self.current_prompt_pos)
 
 
 
 if __name__ == '__main__':
     # Some simple code to test the console widget.
     class MainWindow(wx.Frame):
         def __init__(self, parent, id, title):
             wx.Frame.__init__(self, parent, id, title, size=(300,250))
             self._sizer = wx.BoxSizer(wx.VERTICAL)
             self.console_widget = ConsoleWidget(self)
             self._sizer.Add(self.console_widget, 1, wx.EXPAND)
             self.SetSizer(self._sizer)
             self.SetAutoLayout(1)
             self.Show(True)
 
     app = wx.PySimpleApp()
     w = MainWindow(None, wx.ID_ANY, 'ConsoleWidget')
     w.SetSize((780, 460))
     w.Show()
 
     app.MainLoop()
 
 

IPython/frontend/wx/wx_frontend.py

@@ -1,510 +1,526 b''
 # encoding: utf-8 -*- test-case-name:
 # FIXME: Need to add tests.
 # ipython1.frontend.wx.tests.test_wx_frontend -*-
 
 """Classes to provide a Wx frontend to the
 IPython.kernel.core.interpreter.
 
 This class inherits from ConsoleWidget, that provides a console-like
 widget to provide a text-rendering widget suitable for a terminal.
 """
 
 __docformat__ = "restructuredtext en"
 
 #-------------------------------------------------------------------------------
 #       Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
 
 # Major library imports
 import re
 import __builtin__
 from time import sleep
 import sys
 from threading import Lock
 import string
 
 import wx
 from wx import stc
 
 # Ipython-specific imports.
 from IPython.frontend._process import PipedProcess
 from console_widget import ConsoleWidget
 from IPython.frontend.prefilterfrontend import PrefilterFrontEnd
 
 #-------------------------------------------------------------------------------
 # Constants 
 #-------------------------------------------------------------------------------
 
 _COMPLETE_BUFFER_BG = '#FAFAF1' # Nice green
 _INPUT_BUFFER_BG = '#FDFFD3' # Nice yellow
 _ERROR_BG = '#FFF1F1' # Nice red
 
 _COMPLETE_BUFFER_MARKER = 31
 _ERROR_MARKER = 30
 _INPUT_MARKER = 29
 
 prompt_in1 = \
         '\n\x01\x1b[0;34m\x02In [\x01\x1b[1;34m\x02$number\x01\x1b[0;34m\x02]: \x01\x1b[0m\x02'
 
 prompt_out = \
     '\x01\x1b[0;31m\x02Out[\x01\x1b[1;31m\x02$number\x01\x1b[0;31m\x02]: \x01\x1b[0m\x02'
 
 #-------------------------------------------------------------------------------
 # Classes to implement the Wx frontend
 #-------------------------------------------------------------------------------
 class WxController(ConsoleWidget, PrefilterFrontEnd):
     """Classes to provide a Wx frontend to the
     IPython.kernel.core.interpreter.
 
     This class inherits from ConsoleWidget, that provides a console-like
     widget to provide a text-rendering widget suitable for a terminal.
     """
 
     output_prompt_template = string.Template(prompt_out)
 
     input_prompt_template = string.Template(prompt_in1)
 
     # Print debug info on what is happening to the console.
     debug = False
 
     # The title of the terminal, as captured through the ANSI escape
     # sequences.
     def _set_title(self, title):
             return self.Parent.SetTitle(title)
 
     def _get_title(self):
             return self.Parent.GetTitle()
 
     title = property(_get_title, _set_title)
 
 
     # The buffer being edited.
     # We are duplicating the definition here because of multiple
     # inheritence
     def _set_input_buffer(self, string):
         ConsoleWidget._set_input_buffer(self, string)
         self._colorize_input_buffer()
 
     def _get_input_buffer(self):
         """ Returns the text in current edit buffer.
         """
         return ConsoleWidget._get_input_buffer(self)
 
     input_buffer = property(_get_input_buffer, _set_input_buffer)
 
 
     #--------------------------------------------------------------------------
     # Private Attributes
     #--------------------------------------------------------------------------
 
     # A flag governing the behavior of the input. Can be:
     #
     #       'readline' for readline-like behavior with a prompt 
     #            and an edit buffer.
     #       'raw_input' similar to readline, but triggered by a raw-input
     #           call. Can be used by subclasses to act differently.
     #       'subprocess' for sending the raw input directly to a
     #           subprocess.
     #       'buffering' for buffering of the input, that will be used
     #           when the input state switches back to another state.
     _input_state = 'readline'
 
     # Attribute to store reference to the pipes of a subprocess, if we
     # are running any.
     _running_process = False
 
     # A queue for writing fast streams to the screen without flooding the
     # event loop
     _out_buffer = []
 
     # A lock to lock the _out_buffer to make sure we don't empty it
     # while it is being swapped
     _out_buffer_lock = Lock()
 
+    # The different line markers used to higlight the prompts.
     _markers = dict()
 
     #--------------------------------------------------------------------------
     # Public API
     #--------------------------------------------------------------------------
  
     def __init__(self, parent, id=wx.ID_ANY, pos=wx.DefaultPosition,
-                 size=wx.DefaultSize, style=wx.CLIP_CHILDREN,
+                 size=wx.DefaultSize,
+                 style=wx.CLIP_CHILDREN|wx.WANTS_CHARS,
                  *args, **kwds):
         """ Create Shell instance.
         """
         ConsoleWidget.__init__(self, parent, id, pos, size, style)
         PrefilterFrontEnd.__init__(self, **kwds)
+        
+        # Stick in our own raw_input:
+        self.ipython0.raw_input = self.raw_input
 
         # Marker for complete buffer.
         self.MarkerDefine(_COMPLETE_BUFFER_MARKER, stc.STC_MARK_BACKGROUND,
                                 background=_COMPLETE_BUFFER_BG)
         # Marker for current input buffer.
         self.MarkerDefine(_INPUT_MARKER, stc.STC_MARK_BACKGROUND,
                                 background=_INPUT_BUFFER_BG)
         # Marker for tracebacks.
         self.MarkerDefine(_ERROR_MARKER, stc.STC_MARK_BACKGROUND,
                                 background=_ERROR_BG)
 
         # A time for flushing the write buffer
         BUFFER_FLUSH_TIMER_ID = 100
         self._buffer_flush_timer = wx.Timer(self, BUFFER_FLUSH_TIMER_ID)
         wx.EVT_TIMER(self, BUFFER_FLUSH_TIMER_ID, self._buffer_flush)
 
         if 'debug' in kwds:
             self.debug = kwds['debug']
             kwds.pop('debug')
 
         # Inject self in namespace, for debug
         if self.debug:
             self.shell.user_ns['self'] = self
+        # Inject our own raw_input in namespace
+        self.shell.user_ns['raw_input'] = self.raw_input
 
 
-    def raw_input(self, prompt):
+    def raw_input(self, prompt=''):
         """ A replacement from python's raw_input.
         """
         self.new_prompt(prompt)
         self._input_state = 'raw_input'
         if hasattr(self, '_cursor'):
             del self._cursor 
         self.SetCursor(wx.StockCursor(wx.CURSOR_CROSS))
-        self.waiting = True
         self.__old_on_enter = self._on_enter
+        event_loop = wx.EventLoop()
         def my_on_enter():
-            self.waiting = False
+            event_loop.Exit()
         self._on_enter = my_on_enter
-        # XXX: Busy waiting, ugly.
-        while self.waiting:
-            wx.Yield()
-            sleep(0.1)
+        # XXX: Running a separate event_loop. Ugly.
+        event_loop.Run() 
         self._on_enter = self.__old_on_enter
         self._input_state = 'buffering'
         self._cursor = wx.BusyCursor()
         return self.input_buffer.rstrip('\n')
 
 
     def system_call(self, command_string):
         self._input_state = 'subprocess'
+        event_loop = wx.EventLoop()
+        def _end_system_call():
+            self._input_state = 'buffering'
+            self._running_process = False
+            event_loop.Exit()
+
         self._running_process = PipedProcess(command_string, 
                     out_callback=self.buffered_write,
-                    end_callback = self._end_system_call)
+                    end_callback = _end_system_call)
         self._running_process.start()
-        # XXX: another one of these polling loops to have a blocking
-        # call
-        wx.Yield()
-        while self._running_process:
-            wx.Yield()
-            sleep(0.1)
+        # XXX: Running a separate event_loop. Ugly.
+        event_loop.Run() 
         # Be sure to flush the buffer.
         self._buffer_flush(event=None)
 
 
     def do_calltip(self):
         """ Analyse current and displays useful calltip for it.
         """
         if self.debug:
             print >>sys.__stdout__, "do_calltip" 
         separators =  re.compile('[\s\{\}\[\]\(\)\= ,:]')
         symbol = self.input_buffer
         symbol_string = separators.split(symbol)[-1]
         base_symbol_string = symbol_string.split('.')[0]
         if base_symbol_string in self.shell.user_ns:
             symbol = self.shell.user_ns[base_symbol_string]
         elif base_symbol_string in self.shell.user_global_ns:
             symbol = self.shell.user_global_ns[base_symbol_string]
         elif base_symbol_string in __builtin__.__dict__:
             symbol = __builtin__.__dict__[base_symbol_string]
         else:
             return False
         try:
             for name in symbol_string.split('.')[1:] + ['__doc__']:
                 symbol = getattr(symbol, name)
             self.AutoCompCancel()
-            wx.Yield()
-            self.CallTipShow(self.GetCurrentPos(), symbol)
+            # Check that the symbol can indeed be converted to a string:
+            symbol += ''
+            wx.CallAfter(self.CallTipShow, self.GetCurrentPos(), symbol)
         except:
             # The retrieve symbol couldn't be converted to a string
             pass
 
 
     def _popup_completion(self, create=False):
         """ Updates the popup completion menu if it exists. If create is 
             true, open the menu.
         """
         if self.debug:
-            print >>sys.__stdout__, "_popup_completion", 
+            print >>sys.__stdout__, "_popup_completion" 
         line = self.input_buffer
-        if (self.AutoCompActive() and not line[-1] == '.') \
+        if (self.AutoCompActive() and line and not line[-1] == '.') \
                     or create==True:
             suggestion, completions = self.complete(line)
             offset=0
             if completions:
                 complete_sep =  re.compile('[\s\{\}\[\]\(\)\= ,:]')
                 residual = complete_sep.split(line)[-1]
                 offset = len(residual)
                 self.pop_completion(completions, offset=offset)
                 if self.debug:
                     print >>sys.__stdout__, completions 
 
 
     def buffered_write(self, text):
         """ A write method for streams, that caches the stream in order
             to avoid flooding the event loop.
 
             This can be called outside of the main loop, in separate
             threads.
         """
         self._out_buffer_lock.acquire()
         self._out_buffer.append(text)
         self._out_buffer_lock.release()
         if not self._buffer_flush_timer.IsRunning():
             wx.CallAfter(self._buffer_flush_timer.Start, 
                                         milliseconds=100, oneShot=True)
 
 
     #--------------------------------------------------------------------------
     # LineFrontEnd interface 
     #--------------------------------------------------------------------------
  
     def execute(self, python_string, raw_string=None):
         self._input_state = 'buffering'
         self.CallTipCancel()
         self._cursor = wx.BusyCursor()
         if raw_string is None:
             raw_string = python_string
         end_line = self.current_prompt_line \
                         + max(1,  len(raw_string.split('\n'))-1)
         for i in range(self.current_prompt_line, end_line):
             if i in self._markers:
                 self.MarkerDeleteHandle(self._markers[i])
             self._markers[i] = self.MarkerAdd(i, _COMPLETE_BUFFER_MARKER)
-        # Update the display:
-        wx.Yield()
-        self.GotoPos(self.GetLength())
-        PrefilterFrontEnd.execute(self, python_string, raw_string=raw_string)
+        # Use a callafter to update the display robustly under windows
+        def callback():
+            self.GotoPos(self.GetLength())
+            PrefilterFrontEnd.execute(self, python_string, 
+                                            raw_string=raw_string)
+        wx.CallAfter(callback)
 
     def save_output_hooks(self):    
         self.__old_raw_input = __builtin__.raw_input
         PrefilterFrontEnd.save_output_hooks(self)
 
     def capture_output(self):
-        __builtin__.raw_input = self.raw_input
         self.SetLexer(stc.STC_LEX_NULL)
         PrefilterFrontEnd.capture_output(self)
+        __builtin__.raw_input = self.raw_input
         
     
     def release_output(self):
         __builtin__.raw_input = self.__old_raw_input
         PrefilterFrontEnd.release_output(self)
         self.SetLexer(stc.STC_LEX_PYTHON)
 
 
     def after_execute(self):
         PrefilterFrontEnd.after_execute(self)
         # Clear the wait cursor
         if hasattr(self, '_cursor'):
             del self._cursor
         self.SetCursor(wx.StockCursor(wx.CURSOR_CHAR))
 
 
     def show_traceback(self):
         start_line = self.GetCurrentLine()
         PrefilterFrontEnd.show_traceback(self)
-        wx.Yield()
+        self.ProcessEvent(wx.PaintEvent())
+        #wx.Yield()
         for i in range(start_line, self.GetCurrentLine()):
             self._markers[i] = self.MarkerAdd(i, _ERROR_MARKER)
 
     
     #--------------------------------------------------------------------------
+    # FrontEndBase interface 
+    #--------------------------------------------------------------------------
+    
+    def render_error(self, e):
+        start_line = self.GetCurrentLine()
+        self.write('\n' + e + '\n')
+        for i in range(start_line, self.GetCurrentLine()):
+            self._markers[i] = self.MarkerAdd(i, _ERROR_MARKER)
+
+
+    #--------------------------------------------------------------------------
     # ConsoleWidget interface 
     #--------------------------------------------------------------------------
 
     def new_prompt(self, prompt):
         """ Display a new prompt, and start a new input buffer.
         """
         self._input_state = 'readline'
         ConsoleWidget.new_prompt(self, prompt)
         i = self.current_prompt_line
         self._markers[i] = self.MarkerAdd(i, _INPUT_MARKER)
 
 
     def write(self, *args, **kwargs):
         # Avoid multiple inheritence, be explicit about which
         # parent method class gets called
         ConsoleWidget.write(self, *args, **kwargs)
 
 
     def _on_key_down(self, event, skip=True):
         """ Capture the character events, let the parent
             widget handle them, and put our logic afterward.
         """
         # FIXME: This method needs to be broken down in smaller ones.
         current_line_number = self.GetCurrentLine()
         if event.KeyCode in (ord('c'), ord('C')) and event.ControlDown():
             # Capture Control-C
             if self._input_state == 'subprocess':
                 if self.debug:
                     print >>sys.__stderr__, 'Killing running process'
-                self._running_process.process.kill()
+                if hasattr(self._running_process, 'process'):
+                    self._running_process.process.kill()
             elif self._input_state == 'buffering':
                 if self.debug:
                     print >>sys.__stderr__, 'Raising KeyboardInterrupt'
                 raise KeyboardInterrupt
                 # XXX: We need to make really sure we
                 # get back to a prompt.
         elif self._input_state == 'subprocess' and (
                 ( event.KeyCode<256 and
                         not event.ControlDown() )
                     or 
                 ( event.KeyCode in (ord('d'), ord('D')) and
                   event.ControlDown())):
             #  We are running a process, we redirect keys.
             ConsoleWidget._on_key_down(self, event, skip=skip)
             char = chr(event.KeyCode)
             # Deal with some inconsistency in wx keycodes:
             if char == '\r':
                 char = '\n'
             elif not event.ShiftDown():
                 char = char.lower()
             if event.ControlDown() and event.KeyCode in (ord('d'), ord('D')):
                 char = '\04'
             self._running_process.process.stdin.write(char)
             self._running_process.process.stdin.flush()
-        elif event.KeyCode in (ord('('), 57):
+        elif event.KeyCode in (ord('('), 57, 53):
             # Calltips
             event.Skip()
             self.do_calltip()
         elif self.AutoCompActive() and not event.KeyCode == ord('\t'):
             event.Skip()
             if event.KeyCode in (wx.WXK_BACK, wx.WXK_DELETE): 
                 wx.CallAfter(self._popup_completion, create=True)
             elif not event.KeyCode in (wx.WXK_UP, wx.WXK_DOWN, wx.WXK_LEFT,
                             wx.WXK_RIGHT, wx.WXK_ESCAPE):
                 wx.CallAfter(self._popup_completion)
         else:
             # Up history
             if event.KeyCode == wx.WXK_UP and (
                     ( current_line_number == self.current_prompt_line and
                         event.Modifiers in (wx.MOD_NONE, wx.MOD_WIN) ) 
                     or event.ControlDown() ):
                 new_buffer = self.get_history_previous(
                                             self.input_buffer)
                 if new_buffer is not None:
                     self.input_buffer = new_buffer
                     if self.GetCurrentLine() > self.current_prompt_line:
                         # Go to first line, for seemless history up.
                         self.GotoPos(self.current_prompt_pos)
             # Down history
             elif event.KeyCode == wx.WXK_DOWN and (
                     ( current_line_number == self.LineCount -1 and
                         event.Modifiers in (wx.MOD_NONE, wx.MOD_WIN) ) 
                     or event.ControlDown() ):
                 new_buffer = self.get_history_next()
                 if new_buffer is not None:
                     self.input_buffer = new_buffer
             # Tab-completion
             elif event.KeyCode == ord('\t'):
-                last_line = self.input_buffer.split('\n')[-1]
-                if not re.match(r'^\s*$', last_line):
+                current_line, current_line_number = self.CurLine
+                if not re.match(r'^\s*$', current_line):
                     self.complete_current_input()
                     if self.AutoCompActive():
                         wx.CallAfter(self._popup_completion, create=True)
                 else:
                     event.Skip()
             else:
                 ConsoleWidget._on_key_down(self, event, skip=skip)
 
 
     def _on_key_up(self, event, skip=True):
         """ Called when any key is released.
         """
         if event.KeyCode in (59, ord('.')):
             # Intercepting '.'
             event.Skip()
-            self._popup_completion(create=True)
+            wx.CallAfter(self._popup_completion, create=True)
         else:
             ConsoleWidget._on_key_up(self, event, skip=skip)
 
 
     def _on_enter(self):
         """ Called on return key down, in readline input_state.
         """
         if self.debug:
             print >>sys.__stdout__, repr(self.input_buffer)
         PrefilterFrontEnd._on_enter(self)
 
 
     #--------------------------------------------------------------------------
     # EditWindow API
     #--------------------------------------------------------------------------
 
     def OnUpdateUI(self, event):
         """ Override the OnUpdateUI of the EditWindow class, to prevent 
             syntax highlighting both for faster redraw, and for more
             consistent look and feel.
         """
         if not self._input_state == 'readline':
             ConsoleWidget.OnUpdateUI(self, event)
 
     #--------------------------------------------------------------------------
     # Private API
     #--------------------------------------------------------------------------
  
-    def _end_system_call(self):
-        """ Called at the end of a system call.
-        """
-        self._input_state = 'buffering'
-        self._running_process = False
-
-
     def _buffer_flush(self, event):
         """ Called by the timer to flush the write buffer.
             
             This is always called in the mainloop, by the wx timer.
         """
         self._out_buffer_lock.acquire()
         _out_buffer = self._out_buffer
         self._out_buffer = []
         self._out_buffer_lock.release()
         self.write(''.join(_out_buffer), refresh=False)
 
 
     def _colorize_input_buffer(self):
         """ Keep the input buffer lines at a bright color.
         """
         if not self._input_state in ('readline', 'raw_input'):
             return
         end_line = self.GetCurrentLine()
         if not sys.platform == 'win32':
             end_line += 1
         for i in range(self.current_prompt_line, end_line):
             if i in self._markers:
                 self.MarkerDeleteHandle(self._markers[i])
             self._markers[i] = self.MarkerAdd(i, _INPUT_MARKER)
 
 
 if __name__ == '__main__':
     class MainWindow(wx.Frame):
         def __init__(self, parent, id, title):
             wx.Frame.__init__(self, parent, id, title, size=(300,250))
             self._sizer = wx.BoxSizer(wx.VERTICAL)
             self.shell = WxController(self)
             self._sizer.Add(self.shell, 1, wx.EXPAND)
             self.SetSizer(self._sizer)
             self.SetAutoLayout(1)
             self.Show(True)
 
     app = wx.PySimpleApp()
     frame = MainWindow(None, wx.ID_ANY, 'Ipython')
     frame.shell.SetFocus()
     frame.SetSize((680, 460))
     self = frame.shell
 
     app.MainLoop()
 

IPython/genutils.py

@@ -1,2132 +1,2164 b''
 # -*- coding: utf-8 -*-
 """
 General purpose utilities.
 
 This is a grab-bag of stuff I find useful in most programs I write. Some of
 these things are also convenient when working at the command line.
 
 $Id: genutils.py 2998 2008-01-31 10:06:04Z vivainio $"""
 
 #*****************************************************************************
 #       Copyright (C) 2001-2006 Fernando Perez. <[e-mail]>
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #*****************************************************************************
 
 from IPython import Release
 __author__  = '%s <%s>' % Release.authors['Fernando']
 __license__ = Release.license
 
 #****************************************************************************
 # required modules from the Python standard library
 import __main__
 import commands
 try:
     import doctest
 except ImportError:
     pass
 import os
 import platform
 import re
 import shlex
 import shutil
 import subprocess
 import sys
 import tempfile
 import time
 import types
 import warnings
 
 # Curses and termios are Unix-only modules
 try:
     import curses
     # We need termios as well, so if its import happens to raise, we bail on
     # using curses altogether.
     import termios
 except ImportError:
     USE_CURSES = False
 else:
     # Curses on Solaris may not be complete, so we can't use it there
     USE_CURSES = hasattr(curses,'initscr')
 
 # Other IPython utilities
 import IPython
 from IPython.Itpl import Itpl,itpl,printpl
 from IPython import DPyGetOpt, platutils
 from IPython.generics import result_display
 import IPython.ipapi
 from IPython.external.path import path
 if os.name == "nt":
     from IPython.winconsole import get_console_size
 
 try:
     set
 except:
     from sets import Set as set
 
 
 #****************************************************************************
 # Exceptions
 class Error(Exception):
     """Base class for exceptions in this module."""
     pass
 
 #----------------------------------------------------------------------------
 class IOStream:
     def __init__(self,stream,fallback):
         if not hasattr(stream,'write') or not hasattr(stream,'flush'):
             stream = fallback
         self.stream = stream
         self._swrite = stream.write
         self.flush = stream.flush
 
     def write(self,data):
         try:
             self._swrite(data)
         except:
             try:
                 # print handles some unicode issues which may trip a plain
                 # write() call.  Attempt to emulate write() by using a
                 # trailing comma
                 print >> self.stream, data,
             except:
                 # if we get here, something is seriously broken.
                 print >> sys.stderr, \
                       'ERROR - failed to write data to stream:', self.stream
 
     def close(self):
         pass
 
 
 class IOTerm:
     """ Term holds the file or file-like objects for handling I/O operations.
 
     These are normally just sys.stdin, sys.stdout and sys.stderr but for
     Windows they can can replaced to allow editing the strings before they are
     displayed."""
 
     # In the future, having IPython channel all its I/O operations through
     # this class will make it easier to embed it into other environments which
     # are not a normal terminal (such as a GUI-based shell)
     def __init__(self,cin=None,cout=None,cerr=None):
         self.cin  = IOStream(cin,sys.stdin)
         self.cout = IOStream(cout,sys.stdout)
         self.cerr = IOStream(cerr,sys.stderr)
 
 # Global variable to be used for all I/O
 Term = IOTerm()
 
 import IPython.rlineimpl as readline
 # Remake Term to use the readline i/o facilities
 if sys.platform == 'win32' and readline.have_readline:
 
     Term = IOTerm(cout=readline._outputfile,cerr=readline._outputfile)
 
 
 #****************************************************************************
 # Generic warning/error printer, used by everything else
 def warn(msg,level=2,exit_val=1):
     """Standard warning printer. Gives formatting consistency.
 
     Output is sent to Term.cerr (sys.stderr by default).
 
     Options:
 
     -level(2): allows finer control:
       0 -> Do nothing, dummy function.
       1 -> Print message.
       2 -> Print 'WARNING:' + message. (Default level).
       3 -> Print 'ERROR:' + message.
       4 -> Print 'FATAL ERROR:' + message and trigger a sys.exit(exit_val).
 
     -exit_val (1): exit value returned by sys.exit() for a level 4
     warning. Ignored for all other levels."""
 
     if level>0:
         header = ['','','WARNING: ','ERROR: ','FATAL ERROR: ']
         print >> Term.cerr, '%s%s' % (header[level],msg)
         if level == 4:
             print >> Term.cerr,'Exiting.\n'
             sys.exit(exit_val)
 
 def info(msg):
     """Equivalent to warn(msg,level=1)."""
 
     warn(msg,level=1)
 
 def error(msg):
     """Equivalent to warn(msg,level=3)."""
 
     warn(msg,level=3)
 
 def fatal(msg,exit_val=1):
     """Equivalent to warn(msg,exit_val=exit_val,level=4)."""
 
     warn(msg,exit_val=exit_val,level=4)
 
 #---------------------------------------------------------------------------
 # Debugging routines
 #
 def debugx(expr,pre_msg=''):
     """Print the value of an expression from the caller's frame.
 
     Takes an expression, evaluates it in the caller's frame and prints both
     the given expression and the resulting value (as well as a debug mark
     indicating the name of the calling function.  The input must be of a form
     suitable for eval().
 
     An optional message can be passed, which will be prepended to the printed
     expr->value pair."""
 
     cf = sys._getframe(1)
     print '[DBG:%s] %s%s -> %r' % (cf.f_code.co_name,pre_msg,expr,
                                    eval(expr,cf.f_globals,cf.f_locals))
 
 # deactivate it by uncommenting the following line, which makes it a no-op
 #def debugx(expr,pre_msg=''): pass
 
 #----------------------------------------------------------------------------
 StringTypes = types.StringTypes
 
 # Basic timing functionality
 
 # If possible (Unix), use the resource module instead of time.clock()
 try:
     import resource
     def clocku():
         """clocku() -> floating point number
 
         Return the *USER* CPU time in seconds since the start of the process.
         This is done via a call to resource.getrusage, so it avoids the
         wraparound problems in time.clock()."""
 
         return resource.getrusage(resource.RUSAGE_SELF)[0]
 
     def clocks():
         """clocks() -> floating point number
 
         Return the *SYSTEM* CPU time in seconds since the start of the process.
         This is done via a call to resource.getrusage, so it avoids the
         wraparound problems in time.clock()."""
 
         return resource.getrusage(resource.RUSAGE_SELF)[1]
 
     def clock():
         """clock() -> floating point number
 
         Return the *TOTAL USER+SYSTEM* CPU time in seconds since the start of
         the process.  This is done via a call to resource.getrusage, so it
         avoids the wraparound problems in time.clock()."""
 
         u,s = resource.getrusage(resource.RUSAGE_SELF)[:2]
         return u+s
 
     def clock2():
         """clock2() -> (t_user,t_system)
 
         Similar to clock(), but return a tuple of user/system times."""
         return resource.getrusage(resource.RUSAGE_SELF)[:2]
 
 except ImportError:
     # There is no distinction of user/system time under windows, so we just use
     # time.clock() for everything...
     clocku = clocks = clock = time.clock
     def clock2():
         """Under windows, system CPU time can't be measured.
 
         This just returns clock() and zero."""
         return time.clock(),0.0
 
 def timings_out(reps,func,*args,**kw):
     """timings_out(reps,func,*args,**kw) -> (t_total,t_per_call,output)
 
     Execute a function reps times, return a tuple with the elapsed total
     CPU time in seconds, the time per call and the function's output.
 
     Under Unix, the return value is the sum of user+system time consumed by
     the process, computed via the resource module.  This prevents problems
     related to the wraparound effect which the time.clock() function has.
 
     Under Windows the return value is in wall clock seconds. See the
     documentation for the time module for more details."""
 
     reps = int(reps)
     assert reps >=1, 'reps must be >= 1'
     if reps==1:
         start = clock()
         out = func(*args,**kw)
         tot_time = clock()-start
     else:
         rng = xrange(reps-1) # the last time is executed separately to store output
         start = clock()
         for dummy in rng: func(*args,**kw)
         out = func(*args,**kw)  # one last time
         tot_time = clock()-start
     av_time = tot_time / reps
     return tot_time,av_time,out
 
 def timings(reps,func,*args,**kw):
     """timings(reps,func,*args,**kw) -> (t_total,t_per_call)
 
     Execute a function reps times, return a tuple with the elapsed total CPU
     time in seconds and the time per call. These are just the first two values
     in timings_out()."""
 
     return timings_out(reps,func,*args,**kw)[0:2]
 
 def timing(func,*args,**kw):
     """timing(func,*args,**kw) -> t_total
 
     Execute a function once, return the elapsed total CPU time in
     seconds. This is just the first value in timings_out()."""
 
     return timings_out(1,func,*args,**kw)[0]
 
 #****************************************************************************
 # file and system
 
 def arg_split(s,posix=False):
     """Split a command line's arguments in a shell-like manner.
 
     This is a modified version of the standard library's shlex.split()
     function, but with a default of posix=False for splitting, so that quotes
     in inputs are respected."""
 
     # XXX - there may be unicode-related problems here!!!  I'm not sure that
     # shlex is truly unicode-safe, so it might be necessary to do
     #
     # s = s.encode(sys.stdin.encoding)
     #
     # first, to ensure that shlex gets a normal string.  Input from anyone who
     # knows more about unicode and shlex than I would be good to have here...
     lex = shlex.shlex(s, posix=posix)
     lex.whitespace_split = True
     return list(lex)
 
 def system(cmd,verbose=0,debug=0,header=''):
     """Execute a system command, return its exit status.
 
     Options:
 
     - verbose (0): print the command to be executed.
 
     - debug (0): only print, do not actually execute.
 
     - header (''): Header to print on screen prior to the executed command (it
     is only prepended to the command, no newlines are added).
 
     Note: a stateful version of this function is available through the
     SystemExec class."""
 
     stat = 0
     if verbose or debug: print header+cmd
     sys.stdout.flush()
     if not debug: stat = os.system(cmd)
     return stat
 
 def abbrev_cwd():
     """ Return abbreviated version of cwd, e.g. d:mydir """
     cwd = os.getcwd().replace('\\','/')
     drivepart = ''
     tail = cwd
     if sys.platform == 'win32':
         if len(cwd) < 4:
             return cwd
         drivepart,tail = os.path.splitdrive(cwd)
 
 
     parts = tail.split('/')
     if len(parts) > 2:
         tail = '/'.join(parts[-2:])
 
     return (drivepart + (
         cwd == '/' and '/' or tail))
 
 
 # This function is used by ipython in a lot of places to make system calls.
 # We need it to be slightly different under win32, due to the vagaries of
 # 'network shares'.  A win32 override is below.
 
 def shell(cmd,verbose=0,debug=0,header=''):
     """Execute a command in the system shell, always return None.
 
     Options:
 
     - verbose (0): print the command to be executed.
 
     - debug (0): only print, do not actually execute.
 
     - header (''): Header to print on screen prior to the executed command (it
     is only prepended to the command, no newlines are added).
 
     Note: this is similar to genutils.system(), but it returns None so it can
     be conveniently used in interactive loops without getting the return value
     (typically 0) printed many times."""
 
     stat = 0
     if verbose or debug: print header+cmd
     # flush stdout so we don't mangle python's buffering
     sys.stdout.flush()
 
     if not debug:
         platutils.set_term_title("IPy " + cmd)
         os.system(cmd)
         platutils.set_term_title("IPy " + abbrev_cwd())
 
 # override shell() for win32 to deal with network shares
 if os.name in ('nt','dos'):
 
     shell_ori = shell
 
     def shell(cmd,verbose=0,debug=0,header=''):
         if os.getcwd().startswith(r"\\"):
             path = os.getcwd()
             # change to c drive (cannot be on UNC-share when issuing os.system,
             # as cmd.exe cannot handle UNC addresses)
             os.chdir("c:")
             # issue pushd to the UNC-share and then run the command
             try:
                 shell_ori('"pushd %s&&"'%path+cmd,verbose,debug,header)
             finally:
                 os.chdir(path)
         else:
             shell_ori(cmd,verbose,debug,header)
 
     shell.__doc__ = shell_ori.__doc__
 
 def getoutput(cmd,verbose=0,debug=0,header='',split=0):
     """Dummy substitute for perl's backquotes.
 
     Executes a command and returns the output.
 
     Accepts the same arguments as system(), plus:
 
     - split(0): if true, the output is returned as a list split on newlines.
 
     Note: a stateful version of this function is available through the
     SystemExec class.
 
     This is pretty much deprecated and rarely used,
     genutils.getoutputerror may be what you need.
 
     """
 
     if verbose or debug: print header+cmd
     if not debug:
         output = os.popen(cmd).read()
         # stipping last \n is here for backwards compat.
         if output.endswith('\n'):
             output = output[:-1]
         if split:
             return output.split('\n')
         else:
             return output
 
 def getoutputerror(cmd,verbose=0,debug=0,header='',split=0):
     """Return (standard output,standard error) of executing cmd in a shell.
 
     Accepts the same arguments as system(), plus:
 
     - split(0): if true, each of stdout/err is returned as a list split on
     newlines.
 
     Note: a stateful version of this function is available through the
     SystemExec class."""
 
     if verbose or debug: print header+cmd
     if not cmd:
         if split:
             return [],[]
         else:
             return '',''
     if not debug:
         pin,pout,perr = os.popen3(cmd)
         tout = pout.read().rstrip()
         terr = perr.read().rstrip()
         pin.close()
         pout.close()
         perr.close()
         if split:
             return tout.split('\n'),terr.split('\n')
         else:
             return tout,terr
 
 # for compatibility with older naming conventions
 xsys = system
 bq = getoutput
 
 class SystemExec:
     """Access the system and getoutput functions through a stateful interface.
 
     Note: here we refer to the system and getoutput functions from this
     library, not the ones from the standard python library.
 
     This class offers the system and getoutput functions as methods, but the
     verbose, debug and header parameters can be set for the instance (at
     creation time or later) so that they don't need to be specified on each
     call.
 
     For efficiency reasons, there's no way to override the parameters on a
     per-call basis other than by setting instance attributes. If you need
     local overrides, it's best to directly call system() or getoutput().
 
     The following names are provided as alternate options:
      - xsys: alias to system
      - bq: alias to getoutput
 
     An instance can then be created as:
     >>> sysexec = SystemExec(verbose=1,debug=0,header='Calling: ')
     """
 
     def __init__(self,verbose=0,debug=0,header='',split=0):
         """Specify the instance's values for verbose, debug and header."""
         setattr_list(self,'verbose debug header split')
 
     def system(self,cmd):
         """Stateful interface to system(), with the same keyword parameters."""
 
         system(cmd,self.verbose,self.debug,self.header)
 
     def shell(self,cmd):
         """Stateful interface to shell(), with the same keyword parameters."""
 
         shell(cmd,self.verbose,self.debug,self.header)
 
     xsys = system  # alias
 
     def getoutput(self,cmd):
         """Stateful interface to getoutput()."""
 
         return getoutput(cmd,self.verbose,self.debug,self.header,self.split)
 
     def getoutputerror(self,cmd):
         """Stateful interface to getoutputerror()."""
 
         return getoutputerror(cmd,self.verbose,self.debug,self.header,self.split)
 
     bq = getoutput  # alias
 
 #-----------------------------------------------------------------------------
 def mutex_opts(dict,ex_op):
     """Check for presence of mutually exclusive keys in a dict.
 
     Call: mutex_opts(dict,[[op1a,op1b],[op2a,op2b]...]"""
     for op1,op2 in ex_op:
         if op1 in dict and op2 in dict:
             raise ValueError,'\n*** ERROR in Arguments *** '\
                   'Options '+op1+' and '+op2+' are mutually exclusive.'
 
 #-----------------------------------------------------------------------------
 def get_py_filename(name):
     """Return a valid python filename in the current directory.
 
     If the given name is not a file, it adds '.py' and searches again.
     Raises IOError with an informative message if the file isn't found."""
 
     name = os.path.expanduser(name)
     if not os.path.isfile(name) and not name.endswith('.py'):
         name += '.py'
     if os.path.isfile(name):
         return name
     else:
         raise IOError,'File `%s` not found.' % name
 
 #-----------------------------------------------------------------------------
 def filefind(fname,alt_dirs = None):
     """Return the given filename either in the current directory, if it
     exists, or in a specified list of directories.
 
     ~ expansion is done on all file and directory names.
 
     Upon an unsuccessful search, raise an IOError exception."""
 
     if alt_dirs is None:
         try:
             alt_dirs = get_home_dir()
         except HomeDirError:
             alt_dirs = os.getcwd()
     search = [fname] + list_strings(alt_dirs)
     search = map(os.path.expanduser,search)
     #print 'search list for',fname,'list:',search  # dbg
     fname = search[0]
     if os.path.isfile(fname):
         return fname
     for direc in search[1:]:
         testname = os.path.join(direc,fname)
         #print 'testname',testname  # dbg
         if os.path.isfile(testname):
             return testname
     raise IOError,'File' + `fname` + \
           ' not found in current or supplied directories:' + `alt_dirs`
 
 #----------------------------------------------------------------------------
 def file_read(filename):
     """Read a file and close it.  Returns the file source."""
     fobj = open(filename,'r');
     source = fobj.read();
     fobj.close()
     return source
 
 def file_readlines(filename):
     """Read a file and close it.  Returns the file source using readlines()."""
     fobj = open(filename,'r');
     lines = fobj.readlines();
     fobj.close()
     return lines
 
 #----------------------------------------------------------------------------
 def target_outdated(target,deps):
     """Determine whether a target is out of date.
 
     target_outdated(target,deps) -> 1/0
 
     deps: list of filenames which MUST exist.
     target: single filename which may or may not exist.
 
     If target doesn't exist or is older than any file listed in deps, return
     true, otherwise return false.
     """
     try:
         target_time = os.path.getmtime(target)
     except os.error:
         return 1
     for dep in deps:
         dep_time = os.path.getmtime(dep)
         if dep_time > target_time:
             #print "For target",target,"Dep failed:",dep # dbg
             #print "times (dep,tar):",dep_time,target_time # dbg
             return 1
     return 0
 
 #-----------------------------------------------------------------------------
 def target_update(target,deps,cmd):
     """Update a target with a given command given a list of dependencies.
 
     target_update(target,deps,cmd) -> runs cmd if target is outdated.
 
     This is just a wrapper around target_outdated() which calls the given
     command if target is outdated."""
 
     if target_outdated(target,deps):
         xsys(cmd)
 
 #----------------------------------------------------------------------------
 def unquote_ends(istr):
     """Remove a single pair of quotes from the endpoints of a string."""
 
     if not istr:
         return istr
     if (istr[0]=="'" and istr[-1]=="'") or \
        (istr[0]=='"' and istr[-1]=='"'):
         return istr[1:-1]
     else:
         return istr
 
 #----------------------------------------------------------------------------
 def process_cmdline(argv,names=[],defaults={},usage=''):
     """ Process command-line options and arguments.
 
     Arguments:
 
     - argv: list of arguments, typically sys.argv.
 
     - names: list of option names. See DPyGetOpt docs for details on options
     syntax.
 
     - defaults: dict of default values.
 
     - usage: optional usage notice to print if a wrong argument is passed.
 
     Return a dict of options and a list of free arguments."""
 
     getopt = DPyGetOpt.DPyGetOpt()
     getopt.setIgnoreCase(0)
     getopt.parseConfiguration(names)
 
     try:
         getopt.processArguments(argv)
     except DPyGetOpt.ArgumentError, exc:
         print usage
         warn('"%s"' % exc,level=4)
 
     defaults.update(getopt.optionValues)
     args = getopt.freeValues
 
     return defaults,args
 
 #----------------------------------------------------------------------------
 def optstr2types(ostr):
     """Convert a string of option names to a dict of type mappings.
 
     optstr2types(str) -> {None:'string_opts',int:'int_opts',float:'float_opts'}
 
     This is used to get the types of all the options in a string formatted
     with the conventions of DPyGetOpt. The 'type' None is used for options
     which are strings (they need no further conversion). This function's main
     use is to get a typemap for use with read_dict().
     """
 
     typeconv = {None:'',int:'',float:''}
     typemap = {'s':None,'i':int,'f':float}
     opt_re = re.compile(r'([\w]*)([^:=]*:?=?)([sif]?)')
 
     for w in ostr.split():
         oname,alias,otype = opt_re.match(w).groups()
         if otype == '' or alias == '!':   # simple switches are integers too
             otype = 'i'
         typeconv[typemap[otype]] += oname + ' '
     return typeconv
 
 #----------------------------------------------------------------------------
 def read_dict(filename,type_conv=None,**opt):
     r"""Read a dictionary of key=value pairs from an input file, optionally
     performing conversions on the resulting values.
 
     read_dict(filename,type_conv,**opt) -> dict
 
     Only one value per line is accepted, the format should be
      # optional comments are ignored
      key value\n
 
     Args:
 
       - type_conv: A dictionary specifying which keys need to be converted to
       which types. By default all keys are read as strings. This dictionary
       should have as its keys valid conversion functions for strings
       (int,long,float,complex, or your own).  The value for each key
       (converter) should be a whitespace separated string containing the names
       of all the entries in the file to be converted using that function. For
       keys to be left alone, use None as the conversion function (only needed
       with purge=1, see below).
 
       - opt: dictionary with extra options as below (default in parens)
 
         purge(0): if set to 1, all keys *not* listed in type_conv are purged out
         of the dictionary to be returned. If purge is going to be used, the
         set of keys to be left as strings also has to be explicitly specified
         using the (non-existent) conversion function None.
 
         fs(None): field separator. This is the key/value separator to be used
         when parsing the file. The None default means any whitespace [behavior
         of string.split()].
 
         strip(0): if 1, strip string values of leading/trailinig whitespace.
 
         warn(1): warning level if requested keys are not found in file.
           - 0: silently ignore.
           - 1: inform but proceed.
           - 2: raise KeyError exception.
 
         no_empty(0): if 1, remove keys with whitespace strings as a value.
 
         unique([]): list of keys (or space separated string) which can't be
         repeated. If one such key is found in the file, each new instance
         overwrites the previous one. For keys not listed here, the behavior is
         to make a list of all appearances.
 
     Example:
 
     If the input file test.ini contains (we put it in a string to keep the test
     self-contained):
 
     >>> test_ini = '''\
     ... i 3
     ... x 4.5
     ... y 5.5
     ... s hi ho'''
 
     Then we can use it as follows:
     >>> type_conv={int:'i',float:'x',None:'s'}
 
     >>> d = read_dict(test_ini)
 
     >>> sorted(d.items())
     [('i', '3'), ('s', 'hi ho'), ('x', '4.5'), ('y', '5.5')]
 
     >>> d = read_dict(test_ini,type_conv)
 
     >>> sorted(d.items())
     [('i', 3), ('s', 'hi ho'), ('x', 4.5), ('y', '5.5')]
 
     >>> d = read_dict(test_ini,type_conv,purge=True)
 
     >>> sorted(d.items())
     [('i', 3), ('s', 'hi ho'), ('x', 4.5)]
     """
 
     # starting config
     opt.setdefault('purge',0)
     opt.setdefault('fs',None)  # field sep defaults to any whitespace
     opt.setdefault('strip',0)
     opt.setdefault('warn',1)
     opt.setdefault('no_empty',0)
     opt.setdefault('unique','')
     if type(opt['unique']) in StringTypes:
         unique_keys = qw(opt['unique'])
     elif type(opt['unique']) in (types.TupleType,types.ListType):
         unique_keys = opt['unique']
     else:
         raise ValueError, 'Unique keys must be given as a string, List or Tuple'
 
     dict = {}
 
     # first read in table of values as strings
     if '\n' in filename:
         lines = filename.splitlines()
         file = None
     else:
         file = open(filename,'r')
         lines = file.readlines()
     for line in lines:
         line = line.strip()
         if len(line) and line[0]=='#': continue
         if len(line)>0:
             lsplit = line.split(opt['fs'],1)
             try:
                 key,val = lsplit
             except ValueError:
                 key,val = lsplit[0],''
             key = key.strip()
             if opt['strip']: val = val.strip()
             if val == "''" or val == '""': val = ''
             if opt['no_empty'] and (val=='' or val.isspace()):
                 continue
             # if a key is found more than once in the file, build a list
             # unless it's in the 'unique' list. In that case, last found in file
             # takes precedence. User beware.
             try:
                 if dict[key] and key in unique_keys:
                     dict[key] = val
                 elif type(dict[key]) is types.ListType:
                     dict[key].append(val)
                 else:
                     dict[key] = [dict[key],val]
             except KeyError:
                 dict[key] = val
     # purge if requested
     if opt['purge']:
         accepted_keys = qwflat(type_conv.values())
         for key in dict.keys():
             if key in accepted_keys: continue
             del(dict[key])
     # now convert if requested
     if type_conv==None: return dict
     conversions = type_conv.keys()
     try: conversions.remove(None)
     except: pass
     for convert in conversions:
         for val in qw(type_conv[convert]):
             try:
                 dict[val] = convert(dict[val])
             except KeyError,e:
                 if opt['warn'] == 0:
                     pass
                 elif opt['warn'] == 1:
                     print >>sys.stderr, 'Warning: key',val,\
                           'not found in file',filename
                 elif opt['warn'] == 2:
                     raise KeyError,e
                 else:
                     raise ValueError,'Warning level must be 0,1 or 2'
 
     return dict
 
 #----------------------------------------------------------------------------
 def flag_calls(func):
     """Wrap a function to detect and flag when it gets called.
 
     This is a decorator which takes a function and wraps it in a function with
     a 'called' attribute. wrapper.called is initialized to False.
 
     The wrapper.called attribute is set to False right before each call to the
     wrapped function, so if the call fails it remains False.  After the call
     completes, wrapper.called is set to True and the output is returned.
 
     Testing for truth in wrapper.called allows you to determine if a call to
     func() was attempted and succeeded."""
 
     def wrapper(*args,**kw):
         wrapper.called = False
         out = func(*args,**kw)
         wrapper.called = True
         return out
 
     wrapper.called = False
     wrapper.__doc__ = func.__doc__
     return wrapper
 
 #----------------------------------------------------------------------------
 def dhook_wrap(func,*a,**k):
     """Wrap a function call in a sys.displayhook controller.
 
     Returns a wrapper around func which calls func, with all its arguments and
     keywords unmodified, using the default sys.displayhook.  Since IPython
     modifies sys.displayhook, it breaks the behavior of certain systems that
     rely on the default behavior, notably doctest.
     """
 
     def f(*a,**k):
 
         dhook_s = sys.displayhook
         sys.displayhook = sys.__displayhook__
         try:
             out = func(*a,**k)
         finally:
             sys.displayhook = dhook_s
 
         return out
 
     f.__doc__ = func.__doc__
     return f
 
 #----------------------------------------------------------------------------
 def doctest_reload():
     """Properly reload doctest to reuse it interactively.
 
     This routine:
 
       - reloads doctest
 
       - resets its global 'master' attribute to None, so that multiple uses of
       the module interactively don't produce cumulative reports.
 
       - Monkeypatches its core test runner method to protect it from IPython's
       modified displayhook.  Doctest expects the default displayhook behavior
       deep down, so our modification breaks it completely.  For this reason, a
       hard monkeypatch seems like a reasonable solution rather than asking
       users to manually use a different doctest runner when under IPython."""
 
     import doctest
     reload(doctest)
     doctest.master=None
 
     try:
         doctest.DocTestRunner
     except AttributeError:
         # This is only for python 2.3 compatibility, remove once we move to
         # 2.4 only.
         pass
     else:
         doctest.DocTestRunner.run = dhook_wrap(doctest.DocTestRunner.run)
 
 #----------------------------------------------------------------------------
 class HomeDirError(Error):
     pass
 
 def get_home_dir():
     """Return the closest possible equivalent to a 'home' directory.
 
     We first try $HOME.  Absent that, on NT it's $HOMEDRIVE\$HOMEPATH.
 
     Currently only Posix and NT are implemented, a HomeDirError exception is
     raised for all other OSes. """
 
     isdir = os.path.isdir
     env = os.environ
 
     # first, check py2exe distribution root directory for _ipython.
     # This overrides all. Normally does not exist.
 
     if '\\library.zip\\' in IPython.__file__.lower():
         root, rest = IPython.__file__.lower().split('library.zip')
         if isdir(root + '_ipython'):
             os.environ["IPYKITROOT"] = root.rstrip('\\')
             return root
 
     try:
         homedir = env['HOME']
         if not isdir(homedir):
             # in case a user stuck some string which does NOT resolve to a
             # valid path, it's as good as if we hadn't foud it
             raise KeyError
         return homedir
     except KeyError:
         if os.name == 'posix':
             raise HomeDirError,'undefined $HOME, IPython can not proceed.'
         elif os.name == 'nt':
             # For some strange reason, win9x returns 'nt' for os.name.
             try:
                 homedir = os.path.join(env['HOMEDRIVE'],env['HOMEPATH'])
                 if not isdir(homedir):
                     homedir = os.path.join(env['USERPROFILE'])
                     if not isdir(homedir):
                         raise HomeDirError
                 return homedir
             except:
                 try:
                     # Use the registry to get the 'My Documents' folder.
                     import _winreg as wreg
                     key = wreg.OpenKey(wreg.HKEY_CURRENT_USER,
                                        "Software\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders")
                     homedir = wreg.QueryValueEx(key,'Personal')[0]
                     key.Close()
                     if not isdir(homedir):
                         e = ('Invalid "Personal" folder registry key '
                              'typically "My Documents".\n'
                              'Value: %s\n'
                              'This is not a valid directory on your system.' %
                              homedir)
                         raise HomeDirError(e)
                     return homedir
                 except HomeDirError:
                     raise
                 except:
                     return 'C:\\'
         elif os.name == 'dos':
             # Desperate, may do absurd things in classic MacOS. May work under DOS.
             return 'C:\\'
         else:
             raise HomeDirError,'support for your operating system not implemented.'
 
+
+def get_ipython_dir():
+    """Get the IPython directory for this platform and user.
+    
+    This uses the logic in `get_home_dir` to find the home directory
+    and the adds either .ipython or _ipython to the end of the path.
+    """
+    if os.name == 'posix':
+         ipdir_def = '.ipython'
+    else:
+         ipdir_def = '_ipython'
+    home_dir = get_home_dir()
+    ipdir = os.path.abspath(os.environ.get('IPYTHONDIR',
+                                           os.path.join(home_dir,ipdir_def)))
+    return ipdir
+
+def get_security_dir():
+    """Get the IPython security directory.
+    
+    This directory is the default location for all security related files,
+    including SSL/TLS certificates and FURL files.
+    
+    If the directory does not exist, it is created with 0700 permissions.
+    If it exists, permissions are set to 0700.
+    """
+    security_dir = os.path.join(get_ipython_dir(), 'security')
+    if not os.path.isdir(security_dir):
+        os.mkdir(security_dir, 0700)
+    else:
+        os.chmod(security_dir, 0700)
+    return security_dir
+        
 #****************************************************************************
 # strings and text
 
 class LSString(str):
     """String derivative with a special access attributes.
 
     These are normal strings, but with the special attributes:
 
         .l (or .list) : value as list (split on newlines).
         .n (or .nlstr): original value (the string itself).
         .s (or .spstr): value as whitespace-separated string.
         .p (or .paths): list of path objects
 
     Any values which require transformations are computed only once and
     cached.
 
     Such strings are very useful to efficiently interact with the shell, which
     typically only understands whitespace-separated options for commands."""
 
     def get_list(self):
         try:
             return self.__list
         except AttributeError:
             self.__list = self.split('\n')
             return self.__list
 
     l = list = property(get_list)
 
     def get_spstr(self):
         try:
             return self.__spstr
         except AttributeError:
             self.__spstr = self.replace('\n',' ')
             return self.__spstr
 
     s = spstr = property(get_spstr)
 
     def get_nlstr(self):
         return self
 
     n = nlstr = property(get_nlstr)
 
     def get_paths(self):
         try:
             return self.__paths
         except AttributeError:
             self.__paths = [path(p) for p in self.split('\n') if os.path.exists(p)]
             return self.__paths
 
     p = paths = property(get_paths)
 
 def print_lsstring(arg):
     """ Prettier (non-repr-like) and more informative printer for LSString """
     print "LSString (.p, .n, .l, .s available). Value:"
     print arg
 
 print_lsstring = result_display.when_type(LSString)(print_lsstring)
 
 #----------------------------------------------------------------------------
 class SList(list):
     """List derivative with a special access attributes.
 
     These are normal lists, but with the special attributes:
 
         .l (or .list) : value as list (the list itself).
         .n (or .nlstr): value as a string, joined on newlines.
         .s (or .spstr): value as a string, joined on spaces.
         .p (or .paths): list of path objects
 
     Any values which require transformations are computed only once and
     cached."""
 
     def get_list(self):
         return self
 
     l = list = property(get_list)
 
     def get_spstr(self):
         try:
             return self.__spstr
         except AttributeError:
             self.__spstr = ' '.join(self)
             return self.__spstr
 
     s = spstr = property(get_spstr)
 
     def get_nlstr(self):
         try:
             return self.__nlstr
         except AttributeError:
             self.__nlstr = '\n'.join(self)
             return self.__nlstr
 
     n = nlstr = property(get_nlstr)
 
     def get_paths(self):
         try:
             return self.__paths
         except AttributeError:
             self.__paths = [path(p) for p in self if os.path.exists(p)]
             return self.__paths
 
     p = paths = property(get_paths)
 
     def grep(self, pattern, prune = False, field = None):
         """ Return all strings matching 'pattern' (a regex or callable)
 
         This is case-insensitive. If prune is true, return all items
         NOT matching the pattern.
 
         If field is specified, the match must occur in the specified
         whitespace-separated field.
 
         Examples::
 
             a.grep( lambda x: x.startswith('C') )
             a.grep('Cha.*log', prune=1)
             a.grep('chm', field=-1)
         """
 
         def match_target(s):
             if field is None:
                 return s
             parts = s.split()
             try:
                 tgt = parts[field]
                 return tgt
             except IndexError:
                 return ""
 
         if isinstance(pattern, basestring):
             pred = lambda x : re.search(pattern, x, re.IGNORECASE)
         else:
             pred = pattern
         if not prune:
             return SList([el for el in self if pred(match_target(el))])
         else:
             return SList([el for el in self if not pred(match_target(el))])
     def fields(self, *fields):
         """ Collect whitespace-separated fields from string list
 
         Allows quick awk-like usage of string lists.
 
         Example data (in var a, created by 'a = !ls -l')::
             -rwxrwxrwx  1 ville None      18 Dec 14  2006 ChangeLog
             drwxrwxrwx+ 6 ville None       0 Oct 24 18:05 IPython
 
         a.fields(0) is ['-rwxrwxrwx', 'drwxrwxrwx+']
         a.fields(1,0) is ['1 -rwxrwxrwx', '6 drwxrwxrwx+']
         (note the joining by space).
         a.fields(-1) is ['ChangeLog', 'IPython']
 
         IndexErrors are ignored.
 
         Without args, fields() just split()'s the strings.
         """
         if len(fields) == 0:
             return [el.split() for el in self]
 
         res = SList()
         for el in [f.split() for f in self]:
             lineparts = []
 
             for fd in fields:
                 try:
                     lineparts.append(el[fd])
                 except IndexError:
                     pass
             if lineparts:
                 res.append(" ".join(lineparts))
 
         return res
     def sort(self,field= None,  nums = False):
         """ sort by specified fields (see fields())
 
         Example::
             a.sort(1, nums = True)
 
         Sorts a by second field, in numerical order (so that 21 > 3)
 
         """
 
         #decorate, sort, undecorate
         if field is not None:
             dsu = [[SList([line]).fields(field),  line] for line in self]
         else:
             dsu = [[line,  line] for line in self]
         if nums:
             for i in range(len(dsu)):
                 numstr = "".join([ch for ch in dsu[i][0] if ch.isdigit()])
                 try:
                     n = int(numstr)
                 except ValueError:
                     n = 0;
                 dsu[i][0] = n
 
 
         dsu.sort()
         return SList([t[1] for t in dsu])
 
 def print_slist(arg):
     """ Prettier (non-repr-like) and more informative printer for SList """
     print "SList (.p, .n, .l, .s, .grep(), .fields(), sort() available):"
     if hasattr(arg,  'hideonce') and arg.hideonce:
         arg.hideonce = False
         return
 
     nlprint(arg)
 
 print_slist = result_display.when_type(SList)(print_slist)
 
 
 
 #----------------------------------------------------------------------------
 def esc_quotes(strng):
     """Return the input string with single and double quotes escaped out"""
 
     return strng.replace('"','\\"').replace("'","\\'")
 
 #----------------------------------------------------------------------------
 def make_quoted_expr(s):
     """Return string s in appropriate quotes, using raw string if possible.
 
     Effectively this turns string: cd \ao\ao\
     to: r"cd \ao\ao\_"[:-1]
 
     Note the use of raw string and padding at the end to allow trailing backslash.
 
     """
 
     tail = ''
     tailpadding = ''
     raw  = ''
     if "\\" in s:
         raw = 'r'
         if s.endswith('\\'):
             tail = '[:-1]'
             tailpadding = '_'
     if '"' not in s:
         quote = '"'
     elif "'" not in s:
         quote = "'"
     elif '"""' not in s and not s.endswith('"'):
         quote = '"""'
     elif "'''" not in s and not s.endswith("'"):
         quote = "'''"
     else:
         # give up, backslash-escaped string will do
         return '"%s"' % esc_quotes(s)
     res = raw + quote + s + tailpadding + quote + tail
     return res
 
 
 #----------------------------------------------------------------------------
 def raw_input_multi(header='', ps1='==> ', ps2='..> ',terminate_str = '.'):
     """Take multiple lines of input.
 
     A list with each line of input as a separate element is returned when a
     termination string is entered (defaults to a single '.'). Input can also
     terminate via EOF (^D in Unix, ^Z-RET in Windows).
 
     Lines of input which end in \\ are joined into single entries (and a
     secondary continuation prompt is issued as long as the user terminates
     lines with \\). This allows entering very long strings which are still
     meant to be treated as single entities.
     """
 
     try:
         if header:
             header += '\n'
         lines = [raw_input(header + ps1)]
     except EOFError:
         return []
     terminate = [terminate_str]
     try:
         while lines[-1:] != terminate:
             new_line = raw_input(ps1)
             while new_line.endswith('\\'):
                 new_line = new_line[:-1] + raw_input(ps2)
             lines.append(new_line)
 
         return lines[:-1]  # don't return the termination command
     except EOFError:
         print
         return lines
 
 #----------------------------------------------------------------------------
 def raw_input_ext(prompt='',  ps2='... '):
     """Similar to raw_input(), but accepts extended lines if input ends with \\."""
 
     line = raw_input(prompt)
     while line.endswith('\\'):
         line = line[:-1] + raw_input(ps2)
     return line
 
 #----------------------------------------------------------------------------
 def ask_yes_no(prompt,default=None):
     """Asks a question and returns a boolean (y/n) answer.
 
     If default is given (one of 'y','n'), it is used if the user input is
     empty. Otherwise the question is repeated until an answer is given.
 
     An EOF is treated as the default answer.  If there is no default, an
     exception is raised to prevent infinite loops.
 
     Valid answers are: y/yes/n/no (match is not case sensitive)."""
 
     answers = {'y':True,'n':False,'yes':True,'no':False}
     ans = None
     while ans not in answers.keys():
         try:
             ans = raw_input(prompt+' ').lower()
             if not ans:  # response was an empty string
                 ans = default
         except KeyboardInterrupt:
             pass
         except EOFError:
             if default in answers.keys():
                 ans = default
                 print
             else:
                 raise
 
     return answers[ans]
 
 #----------------------------------------------------------------------------
 def marquee(txt='',width=78,mark='*'):
     """Return the input string centered in a 'marquee'."""
     if not txt:
         return (mark*width)[:width]
     nmark = (width-len(txt)-2)/len(mark)/2
     if nmark < 0: nmark =0
     marks = mark*nmark
     return '%s %s %s' % (marks,txt,marks)
 
 #----------------------------------------------------------------------------
 class EvalDict:
     """
     Emulate a dict which evaluates its contents in the caller's frame.
 
     Usage:
     >>> number = 19
 
     >>> text = "python"
 
     >>> print "%(text.capitalize())s %(number/9.0).1f rules!" % EvalDict()
     Python 2.1 rules!
     """
 
     # This version is due to [e-mail] on c.l.py, and is basically a
     # modified (shorter) version of:
     # http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/66018 by
     # Skip Montanaro ([e-mail]).
 
     def __getitem__(self, name):
         frame = sys._getframe(1)
         return eval(name, frame.f_globals, frame.f_locals)
 
 EvalString = EvalDict  # for backwards compatibility
 #----------------------------------------------------------------------------
 def qw(words,flat=0,sep=None,maxsplit=-1):
     """Similar to Perl's qw() operator, but with some more options.
 
     qw(words,flat=0,sep=' ',maxsplit=-1) -> words.split(sep,maxsplit)
 
     words can also be a list itself, and with flat=1, the output will be
     recursively flattened.
 
     Examples:
 
     >>> qw('1 2')
     ['1', '2']
 
     >>> qw(['a b','1 2',['m n','p q']])
     [['a', 'b'], ['1', '2'], [['m', 'n'], ['p', 'q']]]
 
     >>> qw(['a b','1 2',['m n','p q']],flat=1)
     ['a', 'b', '1', '2', 'm', 'n', 'p', 'q']
     """
 
     if type(words) in StringTypes:
         return [word.strip() for word in words.split(sep,maxsplit)
                 if word and not word.isspace() ]
     if flat:
         return flatten(map(qw,words,[1]*len(words)))
     return map(qw,words)
 
 #----------------------------------------------------------------------------
 def qwflat(words,sep=None,maxsplit=-1):
     """Calls qw(words) in flat mode. It's just a convenient shorthand."""
     return qw(words,1,sep,maxsplit)
 
 #----------------------------------------------------------------------------
 def qw_lol(indata):
     """qw_lol('a b') -> [['a','b']],
     otherwise it's just a call to qw().
 
     We need this to make sure the modules_some keys *always* end up as a
     list of lists."""
 
     if type(indata) in StringTypes:
         return [qw(indata)]
     else:
         return qw(indata)
 
 #-----------------------------------------------------------------------------
 def list_strings(arg):
     """Always return a list of strings, given a string or list of strings
     as input."""
 
     if type(arg) in StringTypes: return [arg]
     else: return arg
 
 #----------------------------------------------------------------------------
 def grep(pat,list,case=1):
     """Simple minded grep-like function.
     grep(pat,list) returns occurrences of pat in list, None on failure.
 
     It only does simple string matching, with no support for regexps. Use the
     option case=0 for case-insensitive matching."""
 
     # This is pretty crude. At least it should implement copying only references
     # to the original data in case it's big. Now it copies the data for output.
     out=[]
     if case:
         for term in list:
             if term.find(pat)>-1: out.append(term)
     else:
         lpat=pat.lower()
         for term in list:
             if term.lower().find(lpat)>-1: out.append(term)
 
     if len(out): return out
     else: return None
 
 #----------------------------------------------------------------------------
 def dgrep(pat,*opts):
     """Return grep() on dir()+dir(__builtins__).
 
     A very common use of grep() when working interactively."""
 
     return grep(pat,dir(__main__)+dir(__main__.__builtins__),*opts)
 
 #----------------------------------------------------------------------------
 def idgrep(pat):
     """Case-insensitive dgrep()"""
 
     return dgrep(pat,0)
 
 #----------------------------------------------------------------------------
 def igrep(pat,list):
     """Synonym for case-insensitive grep."""
 
     return grep(pat,list,case=0)
 
 #----------------------------------------------------------------------------
 def indent(str,nspaces=4,ntabs=0):
     """Indent a string a given number of spaces or tabstops.
 
     indent(str,nspaces=4,ntabs=0) -> indent str by ntabs+nspaces.
     """
     if str is None:
         return
     ind = '\t'*ntabs+' '*nspaces
     outstr = '%s%s' % (ind,str.replace(os.linesep,os.linesep+ind))
     if outstr.endswith(os.linesep+ind):
         return outstr[:-len(ind)]
     else:
         return outstr
 
 #-----------------------------------------------------------------------------
 def native_line_ends(filename,backup=1):
     """Convert (in-place) a file to line-ends native to the current OS.
 
     If the optional backup argument is given as false, no backup of the
     original file is left.  """
 
     backup_suffixes = {'posix':'~','dos':'.bak','nt':'.bak','mac':'.bak'}
 
     bak_filename = filename + backup_suffixes[os.name]
 
     original = open(filename).read()
     shutil.copy2(filename,bak_filename)
     try:
         new = open(filename,'wb')
         new.write(os.linesep.join(original.splitlines()))
         new.write(os.linesep) # ALWAYS put an eol at the end of the file
         new.close()
     except:
         os.rename(bak_filename,filename)
     if not backup:
         try:
             os.remove(bak_filename)
         except:
             pass
 
 #----------------------------------------------------------------------------
 def get_pager_cmd(pager_cmd = None):
     """Return a pager command.
 
     Makes some attempts at finding an OS-correct one."""
 
     if os.name == 'posix':
         default_pager_cmd = 'less -r'  # -r for color control sequences
     elif os.name in ['nt','dos']:
         default_pager_cmd = 'type'
 
     if pager_cmd is None:
         try:
             pager_cmd = os.environ['PAGER']
         except:
             pager_cmd = default_pager_cmd
     return pager_cmd
 
 #-----------------------------------------------------------------------------
 def get_pager_start(pager,start):
     """Return the string for paging files with an offset.
 
     This is the '+N' argument which less and more (under Unix) accept.
     """
 
     if pager in ['less','more']:
         if start:
             start_string = '+' + str(start)
         else:
             start_string = ''
     else:
         start_string = ''
     return start_string
 
 #----------------------------------------------------------------------------
 # (X)emacs on W32 doesn't like to be bypassed with msvcrt.getch()
 if os.name == 'nt' and os.environ.get('TERM','dumb') != 'emacs':
     import msvcrt
     def page_more():
         """ Smart pausing between pages
 
         @return:    True if need print more lines, False if quit
         """
         Term.cout.write('---Return to continue, q to quit--- ')
         ans = msvcrt.getch()
         if ans in ("q", "Q"):
             result = False
         else:
             result = True
         Term.cout.write("\b"*37 + " "*37 + "\b"*37)
         return result
 else:
     def page_more():
         ans = raw_input('---Return to continue, q to quit--- ')
         if ans.lower().startswith('q'):
             return False
         else:
             return True
 
 esc_re = re.compile(r"(\x1b[^m]+m)")
 
 def page_dumb(strng,start=0,screen_lines=25):
     """Very dumb 'pager' in Python, for when nothing else works.
 
     Only moves forward, same interface as page(), except for pager_cmd and
     mode."""
 
     out_ln  = strng.splitlines()[start:]
     screens = chop(out_ln,screen_lines-1)
     if len(screens) == 1:
         print >>Term.cout, os.linesep.join(screens[0])
     else:
         last_escape = ""
         for scr in screens[0:-1]:
             hunk = os.linesep.join(scr)
             print >>Term.cout, last_escape + hunk
             if not page_more():
                 return
             esc_list = esc_re.findall(hunk)
             if len(esc_list) > 0:
                 last_escape = esc_list[-1]
         print >>Term.cout, last_escape + os.linesep.join(screens[-1])
 
 #----------------------------------------------------------------------------
 def page(strng,start=0,screen_lines=0,pager_cmd = None):
     """Print a string, piping through a pager after a certain length.
 
     The screen_lines parameter specifies the number of *usable* lines of your
     terminal screen (total lines minus lines you need to reserve to show other
     information).
 
     If you set screen_lines to a number <=0, page() will try to auto-determine
     your screen size and will only use up to (screen_size+screen_lines) for
     printing, paging after that. That is, if you want auto-detection but need
     to reserve the bottom 3 lines of the screen, use screen_lines = -3, and for
     auto-detection without any lines reserved simply use screen_lines = 0.
 
     If a string won't fit in the allowed lines, it is sent through the
     specified pager command. If none given, look for PAGER in the environment,
     and ultimately default to less.
 
     If no system pager works, the string is sent through a 'dumb pager'
     written in python, very simplistic.
     """
 
     # Some routines may auto-compute start offsets incorrectly and pass a
     # negative value.  Offset to 0 for robustness.
     start = max(0,start)
 
     # first, try the hook
     ip = IPython.ipapi.get()
     if ip:
         try:
             ip.IP.hooks.show_in_pager(strng)
             return
         except IPython.ipapi.TryNext:
             pass
 
     # Ugly kludge, but calling curses.initscr() flat out crashes in emacs
     TERM = os.environ.get('TERM','dumb')
     if TERM in ['dumb','emacs'] and os.name != 'nt':
         print strng
         return
     # chop off the topmost part of the string we don't want to see
     str_lines = strng.split(os.linesep)[start:]
     str_toprint = os.linesep.join(str_lines)
     num_newlines = len(str_lines)
     len_str = len(str_toprint)
 
     # Dumb heuristics to guesstimate number of on-screen lines the string
     # takes.  Very basic, but good enough for docstrings in reasonable
     # terminals. If someone later feels like refining it, it's not hard.
     numlines = max(num_newlines,int(len_str/80)+1)
 
     if os.name == "nt":
         screen_lines_def = get_console_size(defaulty=25)[1]
     else:
         screen_lines_def = 25 # default value if we can't auto-determine
 
     # auto-determine screen size
     if screen_lines <= 0:
         if TERM=='xterm':
             use_curses = USE_CURSES
         else:
             # curses causes problems on many terminals other than xterm.
             use_curses = False
         if use_curses:
             # There is a bug in curses, where *sometimes* it fails to properly
             # initialize, and then after the endwin() call is made, the
             # terminal is left in an unusable state.  Rather than trying to
             # check everytime for this (by requesting and comparing termios
             # flags each time), we just save the initial terminal state and
             # unconditionally reset it every time.  It's cheaper than making
             # the checks.
             term_flags = termios.tcgetattr(sys.stdout)
             scr = curses.initscr()
             screen_lines_real,screen_cols = scr.getmaxyx()
             curses.endwin()
             # Restore terminal state in case endwin() didn't.
             termios.tcsetattr(sys.stdout,termios.TCSANOW,term_flags)
             # Now we have what we needed: the screen size in rows/columns
             screen_lines += screen_lines_real
             #print '***Screen size:',screen_lines_real,'lines x',\
             #screen_cols,'columns.' # dbg
         else:
             screen_lines += screen_lines_def
 
     #print 'numlines',numlines,'screenlines',screen_lines  # dbg
     if numlines <= screen_lines :
         #print '*** normal print'  # dbg
         print >>Term.cout, str_toprint
     else:
         # Try to open pager and default to internal one if that fails.
         # All failure modes are tagged as 'retval=1', to match the return
         # value of a failed system command.  If any intermediate attempt
         # sets retval to 1, at the end we resort to our own page_dumb() pager.
         pager_cmd = get_pager_cmd(pager_cmd)
         pager_cmd += ' ' + get_pager_start(pager_cmd,start)
         if os.name == 'nt':
             if pager_cmd.startswith('type'):
                 # The default WinXP 'type' command is failing on complex strings.
                 retval = 1
             else:
                 tmpname = tempfile.mktemp('.txt')
                 tmpfile = file(tmpname,'wt')
                 tmpfile.write(strng)
                 tmpfile.close()
                 cmd = "%s < %s" % (pager_cmd,tmpname)
                 if os.system(cmd):
                   retval = 1
                 else:
                   retval = None
                 os.remove(tmpname)
         else:
             try:
                 retval = None
                 # if I use popen4, things hang. No idea why.
                 #pager,shell_out = os.popen4(pager_cmd)
                 pager = os.popen(pager_cmd,'w')
                 pager.write(strng)
                 pager.close()
                 retval = pager.close()  # success returns None
             except IOError,msg:  # broken pipe when user quits
                 if msg.args == (32,'Broken pipe'):
                     retval = None
                 else:
                     retval = 1
             except OSError:
                 # Other strange problems, sometimes seen in Win2k/cygwin
                 retval = 1
         if retval is not None:
             page_dumb(strng,screen_lines=screen_lines)
 
 #----------------------------------------------------------------------------
 def page_file(fname,start = 0, pager_cmd = None):
     """Page a file, using an optional pager command and starting line.
     """
 
     pager_cmd = get_pager_cmd(pager_cmd)
     pager_cmd += ' ' + get_pager_start(pager_cmd,start)
 
     try:
         if os.environ['TERM'] in ['emacs','dumb']:
             raise EnvironmentError
         xsys(pager_cmd + ' ' + fname)
     except:
         try:
             if start > 0:
                 start -= 1
             page(open(fname).read(),start)
         except:
             print 'Unable to show file',`fname`
 
 
 #----------------------------------------------------------------------------
 def snip_print(str,width = 75,print_full = 0,header = ''):
     """Print a string snipping the midsection to fit in width.
 
     print_full: mode control:
       - 0: only snip long strings
       - 1: send to page() directly.
       - 2: snip long strings and ask for full length viewing with page()
     Return 1 if snipping was necessary, 0 otherwise."""
 
     if print_full == 1:
         page(header+str)
         return 0
 
     print header,
     if len(str) < width:
         print str
         snip = 0
     else:
         whalf = int((width -5)/2)
         print str[:whalf] + ' <...> ' + str[-whalf:]
         snip = 1
     if snip and print_full == 2:
         if raw_input(header+' Snipped. View (y/n)? [N]').lower() == 'y':
             page(str)
     return snip
 
 #****************************************************************************
 # lists, dicts and structures
 
 def belong(candidates,checklist):
     """Check whether a list of items appear in a given list of options.
 
     Returns a list of 1 and 0, one for each candidate given."""
 
     return [x in checklist for x in candidates]
 
 #----------------------------------------------------------------------------
 def uniq_stable(elems):
     """uniq_stable(elems) -> list
 
     Return from an iterable, a list of all the unique elements in the input,
     but maintaining the order in which they first appear.
 
     A naive solution to this problem which just makes a dictionary with the
     elements as keys fails to respect the stability condition, since
     dictionaries are unsorted by nature.
 
     Note: All elements in the input must be valid dictionary keys for this
     routine to work, as it internally uses a dictionary for efficiency
     reasons."""
 
     unique = []
     unique_dict = {}
     for nn in elems:
         if nn not in unique_dict:
             unique.append(nn)
             unique_dict[nn] = None
     return unique
 
 #----------------------------------------------------------------------------
 class NLprinter:
     """Print an arbitrarily nested list, indicating index numbers.
 
     An instance of this class called nlprint is available and callable as a
     function.
 
     nlprint(list,indent=' ',sep=': ') -> prints indenting each level by 'indent'
     and using 'sep' to separate the index from the value. """
 
     def __init__(self):
         self.depth = 0
 
     def __call__(self,lst,pos='',**kw):
         """Prints the nested list numbering levels."""
         kw.setdefault('indent',' ')
         kw.setdefault('sep',': ')
         kw.setdefault('start',0)
         kw.setdefault('stop',len(lst))
         # we need to remove start and stop from kw so they don't propagate
         # into a recursive call for a nested list.
         start = kw['start']; del kw['start']
         stop = kw['stop']; del kw['stop']
         if self.depth == 0 and 'header' in kw.keys():
             print kw['header']
 
         for idx in range(start,stop):
             elem = lst[idx]
             if type(elem)==type([]):
                 self.depth += 1
                 self.__call__(elem,itpl('$pos$idx,'),**kw)
                 self.depth -= 1
             else:
                 printpl(kw['indent']*self.depth+'$pos$idx$kw["sep"]$elem')
 
 nlprint = NLprinter()
 #----------------------------------------------------------------------------
 def all_belong(candidates,checklist):
     """Check whether a list of items ALL appear in a given list of options.
 
     Returns a single 1 or 0 value."""
 
     return 1-(0 in [x in checklist for x in candidates])
 
 #----------------------------------------------------------------------------
 def sort_compare(lst1,lst2,inplace = 1):
     """Sort and compare two lists.
 
     By default it does it in place, thus modifying the lists. Use inplace = 0
     to avoid that (at the cost of temporary copy creation)."""
     if not inplace:
         lst1 = lst1[:]
         lst2 = lst2[:]
     lst1.sort(); lst2.sort()
     return lst1 == lst2
 
 #----------------------------------------------------------------------------
 def list2dict(lst):
     """Takes a list of (key,value) pairs and turns it into a dict."""
 
     dic = {}
     for k,v in lst: dic[k] = v
     return dic
 
 #----------------------------------------------------------------------------
 def list2dict2(lst,default=''):
     """Takes a list and turns it into a dict.
     Much slower than list2dict, but more versatile. This version can take
     lists with sublists of arbitrary length (including sclars)."""
 
     dic = {}
     for elem in lst:
         if type(elem) in (types.ListType,types.TupleType):
             size = len(elem)
             if  size == 0:
                 pass
             elif size == 1:
                 dic[elem] = default
             else:
                 k,v = elem[0], elem[1:]
                 if len(v) == 1: v = v[0]
                 dic[k] = v
         else:
             dic[elem] = default
     return dic
 
 #----------------------------------------------------------------------------
 def flatten(seq):
     """Flatten a list of lists (NOT recursive, only works for 2d lists)."""
 
     return [x for subseq in seq for x in subseq]
 
 #----------------------------------------------------------------------------
 def get_slice(seq,start=0,stop=None,step=1):
     """Get a slice of a sequence with variable step. Specify start,stop,step."""
     if stop == None:
         stop = len(seq)
     item = lambda i: seq[i]
     return map(item,xrange(start,stop,step))
 
 #----------------------------------------------------------------------------
 def chop(seq,size):
     """Chop a sequence into chunks of the given size."""
     chunk = lambda i: seq[i:i+size]
     return map(chunk,xrange(0,len(seq),size))
 
 #----------------------------------------------------------------------------
 # with is a keyword as of python 2.5, so this function is renamed to withobj
 # from its old 'with' name.
 def with_obj(object, **args):
     """Set multiple attributes for an object, similar to Pascal's with.
 
     Example:
     with_obj(jim,
              born = 1960,
              haircolour = 'Brown',
              eyecolour = 'Green')
 
     Credit: Greg Ewing, in
     http://mail.python.org/pipermail/python-list/2001-May/040703.html.
 
     NOTE: up until IPython 0.7.2, this was called simply 'with', but 'with'
     has become a keyword for Python 2.5, so we had to rename it."""
 
     object.__dict__.update(args)
 
 #----------------------------------------------------------------------------
 def setattr_list(obj,alist,nspace = None):
     """Set a list of attributes for an object taken from a namespace.
 
     setattr_list(obj,alist,nspace) -> sets in obj all the attributes listed in
     alist with their values taken from nspace, which must be a dict (something
     like locals() will often do) If nspace isn't given, locals() of the
     *caller* is used, so in most cases you can omit it.
 
     Note that alist can be given as a string, which will be automatically
     split into a list on whitespace. If given as a list, it must be a list of
     *strings* (the variable names themselves), not of variables."""
 
     # this grabs the local variables from the *previous* call frame -- that is
     # the locals from the function that called setattr_list().
     # - snipped from weave.inline()
     if nspace is None:
         call_frame = sys._getframe().f_back
         nspace = call_frame.f_locals
 
     if type(alist) in StringTypes:
         alist = alist.split()
     for attr in alist:
         val = eval(attr,nspace)
         setattr(obj,attr,val)
 
 #----------------------------------------------------------------------------
 def getattr_list(obj,alist,*args):
     """getattr_list(obj,alist[, default]) -> attribute list.
 
     Get a list of named attributes for an object. When a default argument is
     given, it is returned when the attribute doesn't exist; without it, an
     exception is raised in that case.
 
     Note that alist can be given as a string, which will be automatically
     split into a list on whitespace. If given as a list, it must be a list of
     *strings* (the variable names themselves), not of variables."""
 
     if type(alist) in StringTypes:
         alist = alist.split()
     if args:
         if len(args)==1:
             default = args[0]
             return map(lambda attr: getattr(obj,attr,default),alist)
         else:
             raise ValueError,'getattr_list() takes only one optional argument'
     else:
         return map(lambda attr: getattr(obj,attr),alist)
 
 #----------------------------------------------------------------------------
 def map_method(method,object_list,*argseq,**kw):
     """map_method(method,object_list,*args,**kw) -> list
 
     Return a list of the results of applying the methods to the items of the
     argument sequence(s).  If more than one sequence is given, the method is
     called with an argument list consisting of the corresponding item of each
     sequence. All sequences must be of the same length.
 
     Keyword arguments are passed verbatim to all objects called.
 
     This is Python code, so it's not nearly as fast as the builtin map()."""
 
     out_list = []
     idx = 0
     for object in object_list:
         try:
             handler = getattr(object, method)
         except AttributeError:
             out_list.append(None)
         else:
             if argseq:
                 args = map(lambda lst:lst[idx],argseq)
                 #print 'ob',object,'hand',handler,'ar',args # dbg
                 out_list.append(handler(args,**kw))
             else:
                 out_list.append(handler(**kw))
         idx += 1
     return out_list
 
 #----------------------------------------------------------------------------
 def get_class_members(cls):
     ret = dir(cls)
     if hasattr(cls,'__bases__'):
         for base in cls.__bases__:
             ret.extend(get_class_members(base))
     return ret
 
 #----------------------------------------------------------------------------
 def dir2(obj):
     """dir2(obj) -> list of strings
 
     Extended version of the Python builtin dir(), which does a few extra
     checks, and supports common objects with unusual internals that confuse
     dir(), such as Traits and PyCrust.
 
     This version is guaranteed to return only a list of true strings, whereas
     dir() returns anything that objects inject into themselves, even if they
     are later not really valid for attribute access (many extension libraries
     have such bugs).
     """
 
     # Start building the attribute list via dir(), and then complete it
     # with a few extra special-purpose calls.
     words = dir(obj)
 
     if hasattr(obj,'__class__'):
         words.append('__class__')
         words.extend(get_class_members(obj.__class__))
     #if '__base__' in words: 1/0
 
     # Some libraries (such as traits) may introduce duplicates, we want to
     # track and clean this up if it happens
     may_have_dupes = False
 
     # this is the 'dir' function for objects with Enthought's traits
     if hasattr(obj, 'trait_names'):
         try:
             words.extend(obj.trait_names())
             may_have_dupes = True
         except TypeError:
             # This will happen if `obj` is a class and not an instance.
             pass
 
     # Support for PyCrust-style _getAttributeNames magic method.
     if hasattr(obj, '_getAttributeNames'):
         try:
             words.extend(obj._getAttributeNames())
             may_have_dupes = True
         except TypeError:
             # `obj` is a class and not an instance.  Ignore
             # this error.
             pass
 
     if may_have_dupes:
         # eliminate possible duplicates, as some traits may also
         # appear as normal attributes in the dir() call.
         words = list(set(words))
         words.sort()
 
     # filter out non-string attributes which may be stuffed by dir() calls
     # and poor coding in third-party modules
     return [w for w in words if isinstance(w, basestring)]
 
 #----------------------------------------------------------------------------
 def import_fail_info(mod_name,fns=None):
     """Inform load failure for a module."""
 
     if fns == None:
         warn("Loading of %s failed.\n" % (mod_name,))
     else:
         warn("Loading of %s from %s failed.\n" % (fns,mod_name))
 
 #----------------------------------------------------------------------------
 # Proposed popitem() extension, written as a method
 
 
 class NotGiven: pass
 
 def popkey(dct,key,default=NotGiven):
     """Return dct[key] and delete dct[key].
 
     If default is given, return it if dct[key] doesn't exist, otherwise raise
     KeyError.  """
 
     try:
         val = dct[key]
     except KeyError:
         if default is NotGiven:
             raise
         else:
             return default
     else:
         del dct[key]
         return val
 
 def wrap_deprecated(func, suggest = '<nothing>'):
     def newFunc(*args, **kwargs):
         warnings.warn("Call to deprecated function %s, use %s instead" %
                       ( func.__name__, suggest),
                       category=DeprecationWarning,
                       stacklevel = 2)
         return func(*args, **kwargs)
     return newFunc
 
 
 def _num_cpus_unix():
     """Return the number of active CPUs on a Unix system."""
     return os.sysconf("SC_NPROCESSORS_ONLN")
 
 
 def _num_cpus_darwin():
     """Return the number of active CPUs on a Darwin system."""
     p = subprocess.Popen(['sysctl','-n','hw.ncpu'],stdout=subprocess.PIPE)
     return p.stdout.read()
 
 
 def _num_cpus_windows():
     """Return the number of active CPUs on a Windows system."""
     return os.environ.get("NUMBER_OF_PROCESSORS")
 
 
 def num_cpus():
    """Return the effective number of CPUs in the system as an integer.
 
    This cross-platform function makes an attempt at finding the total number of
    available CPUs in the system, as returned by various underlying system and
    python calls.
 
    If it can't find a sensible answer, it returns 1 (though an error *may* make
    it return a large positive number that's actually incorrect).
    """
 
    # Many thanks to the Parallel Python project (http://www.parallelpython.com)
    # for the names of the keys we needed to look up for this function.  This
    # code was inspired by their equivalent function.
 
    ncpufuncs = {'Linux':_num_cpus_unix,
                 'Darwin':_num_cpus_darwin,
                 'Windows':_num_cpus_windows,
                 # On Vista, python < 2.5.2 has a bug and returns 'Microsoft'
                 # See http://bugs.python.org/issue1082 for details.
                 'Microsoft':_num_cpus_windows,
                 }
 
    ncpufunc = ncpufuncs.get(platform.system(),
                             # default to unix version (Solaris, AIX, etc)
                             _num_cpus_unix)
 
    try:
        ncpus = max(1,int(ncpufunc()))
    except:
        ncpus = 1
    return ncpus
 
 #*************************** end of file <genutils.py> **********************

IPython/kernel/config/__init__.py

@@ -1,125 +1,123 b''
 # encoding: utf-8
 
 """Default kernel configuration."""
 
 __docformat__ = "restructuredtext en"
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
 
+from os.path import join as pjoin
+
 from IPython.external.configobj import ConfigObj
 from IPython.config.api import ConfigObjManager
-from IPython.config.cutils import get_ipython_dir
+from IPython.genutils import get_ipython_dir, get_security_dir
 
 default_kernel_config = ConfigObj()
 
-try:
-    ipython_dir = get_ipython_dir() + '/'
-except:
-    # This will defaults to the cwd
-    ipython_dir = ''
+security_dir = get_security_dir()
 
 #-------------------------------------------------------------------------------
 # Engine Configuration
 #-------------------------------------------------------------------------------
 
 engine_config = dict(
     logfile = '',    # Empty means log to stdout
-    furl_file = ipython_dir + 'ipcontroller-engine.furl'
+    furl_file = pjoin(security_dir, 'ipcontroller-engine.furl')
 )
 
 #-------------------------------------------------------------------------------
 # MPI Configuration
 #-------------------------------------------------------------------------------
 
 mpi_config = dict(
     mpi4py = """from mpi4py import MPI as mpi
 mpi.size = mpi.COMM_WORLD.Get_size()
 mpi.rank = mpi.COMM_WORLD.Get_rank()
 """,
     pytrilinos = """from PyTrilinos import Epetra
 class SimpleStruct:
     pass
 mpi = SimpleStruct()
 mpi.rank = 0
 mpi.size = 0
 """,
     default = ''
 )
 
 #-------------------------------------------------------------------------------
 # Controller Configuration
 #-------------------------------------------------------------------------------
 
 controller_config = dict(
 
     logfile = '',    # Empty means log to stdout
     import_statement = '',
 
     engine_tub = dict(
         ip = '',         # Empty string means all interfaces
         port = 0,        # 0 means pick a port for me
         location = '',    # Empty string means try to set automatically
         secure = True,
-        cert_file = ipython_dir + 'ipcontroller-engine.pem',
+        cert_file = pjoin(security_dir, 'ipcontroller-engine.pem'),
     ),
     engine_fc_interface = 'IPython.kernel.enginefc.IFCControllerBase',
-    engine_furl_file = ipython_dir + 'ipcontroller-engine.furl',
+    engine_furl_file = pjoin(security_dir, 'ipcontroller-engine.furl'),
     
     controller_interfaces = dict(
         # multiengine = dict(
         #     controller_interface = 'IPython.kernel.multiengine.IMultiEngine',
         #     fc_interface = 'IPython.kernel.multienginefc.IFCMultiEngine',
         #     furl_file = 'ipcontroller-mec.furl'
         # ),
         task = dict(
             controller_interface = 'IPython.kernel.task.ITaskController',
             fc_interface = 'IPython.kernel.taskfc.IFCTaskController',
-            furl_file = ipython_dir + 'ipcontroller-tc.furl'
+            furl_file = pjoin(security_dir, 'ipcontroller-tc.furl')
         ),
         multiengine = dict(
             controller_interface = 'IPython.kernel.multiengine.IMultiEngine',
             fc_interface = 'IPython.kernel.multienginefc.IFCSynchronousMultiEngine',
-            furl_file = ipython_dir + 'ipcontroller-mec.furl'
+            furl_file = pjoin(security_dir, 'ipcontroller-mec.furl')
         )
     ),
 
     client_tub = dict(
         ip = '',         # Empty string means all interfaces
         port = 0,        # 0 means pick a port for me
         location = '',    # Empty string means try to set automatically
         secure = True,
-        cert_file = ipython_dir + 'ipcontroller-client.pem'
+        cert_file = pjoin(security_dir, 'ipcontroller-client.pem')
     )
 )
 
 #-------------------------------------------------------------------------------
 # Client Configuration
 #-------------------------------------------------------------------------------
 
 client_config = dict(
     client_interfaces = dict(
         task = dict(
-            furl_file = ipython_dir + 'ipcontroller-tc.furl'
+            furl_file = pjoin(security_dir, 'ipcontroller-tc.furl')
         ),
         multiengine = dict(
-            furl_file = ipython_dir + 'ipcontroller-mec.furl'
+            furl_file = pjoin(security_dir, 'ipcontroller-mec.furl')
         )
     )
 )
 
 default_kernel_config['engine'] = engine_config
 default_kernel_config['mpi'] = mpi_config
 default_kernel_config['controller'] = controller_config
 default_kernel_config['client'] = client_config
 
 
 config_manager = ConfigObjManager(default_kernel_config, 'IPython.kernel.ini')
  No newline at end of file

IPython/kernel/controllerservice.py

@@ -1,376 +1,376 b''
 # encoding: utf-8
 # -*- test-case-name: IPython.kernel.test.test_controllerservice -*-
 
 """A Twisted Service for the IPython Controller.
 
 The IPython Controller:
 
 * Listens for Engines to connect and then manages access to those engines.
 * Listens for clients and passes commands from client to the Engines.
 * Exposes an asynchronous interfaces to the Engines which themselves can block.
 * Acts as a gateway to the Engines.
 
 The design of the controller is somewhat abstract to allow flexibility in how 
 the controller is presented to clients.  This idea is that there is a basic
 ControllerService class that allows engines to connect to it.  But, this 
 basic class has no client interfaces.  To expose client interfaces developers
 provide an adapter that makes the ControllerService look like something.  For 
 example, one client interface might support task farming and another might
 support interactive usage.  The important thing is that by using interfaces
 and adapters, a single controller can be accessed from multiple interfaces.
 Furthermore, by adapting various client interfaces to various network
 protocols, each client interface can be exposed to multiple network protocols.
 See multiengine.py for an example of how to adapt the ControllerService
 to a client interface.
 """
 
 __docformat__ = "restructuredtext en"
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
 
 import os, sys
 
 from twisted.application import service
 from twisted.internet import defer, reactor
 from twisted.python import log, components
 from zope.interface import Interface, implements, Attribute
 import zope.interface as zi
 
 from IPython.kernel.engineservice import \
     IEngineCore, \
     IEngineSerialized, \
     IEngineQueued
     
-from IPython.config import cutils
+from IPython.genutils import get_ipython_dir
 from IPython.kernel import codeutil
 
 #-------------------------------------------------------------------------------
 # Interfaces for the Controller
 #-------------------------------------------------------------------------------
 
 class IControllerCore(Interface):
     """Basic methods any controller must have.
     
     This is basically the aspect of the controller relevant to the 
     engines and does not assume anything about how the engines will
     be presented to a client.
     """
         
     engines = Attribute("A dict of engine ids and engine instances.")
         
     def register_engine(remoteEngine, id=None, ip=None, port=None, 
         pid=None):
         """Register new remote engine.
         
         The controller can use the ip, port, pid of the engine to do useful things
         like kill the engines.
         
         :Parameters:
             remoteEngine
                 An implementer of IEngineCore, IEngineSerialized and IEngineQueued.
             id : int
                 Requested id.
             ip : str
                 IP address the engine is running on.
             port : int
                 Port the engine is on.
             pid : int
                 pid of the running engine.
         
         :Returns: A dict of {'id':id} and possibly other key, value pairs.
         """
     
     def unregister_engine(id):
         """Handle a disconnecting engine.
         
         :Parameters:
             id
                 The integer engine id of the engine to unregister.
         """
         
     def on_register_engine_do(f, includeID, *args, **kwargs):
         """Call ``f(*args, **kwargs)`` when an engine is registered.
         
         :Parameters:
             includeID : int
                 If True the first argument to f will be the id of the engine.
         """
             
     def on_unregister_engine_do(f, includeID, *args, **kwargs):
         """Call ``f(*args, **kwargs)`` when an engine is unregistered.
         
         :Parameters:
             includeID : int
                 If True the first argument to f will be the id of the engine.
         """
     
     def on_register_engine_do_not(f):
         """Stop calling f on engine registration"""
     
     def on_unregister_engine_do_not(f):
         """Stop calling f on engine unregistration"""
         
     def on_n_engines_registered_do(n, f, *arg, **kwargs):
         """Call f(*args, **kwargs) the first time the nth engine registers."""
                     
 class IControllerBase(IControllerCore):
     """The basic controller interface."""
     pass 
 
 
 #-------------------------------------------------------------------------------
 # Implementation of the ControllerService
 #-------------------------------------------------------------------------------
 
 class ControllerService(object, service.Service):
     """A basic Controller represented as a Twisted Service.
     
     This class doesn't implement any client notification mechanism.  That
     is up to adapted subclasses.
     """
     
     # I also pick up the IService interface by inheritance from service.Service
     implements(IControllerBase)
     name = 'ControllerService'
     
     def __init__(self, maxEngines=511, saveIDs=False):
         self.saveIDs = saveIDs
         self.engines = {}
         self.availableIDs = range(maxEngines,-1,-1)   # [511,...,0]
         self._onRegister = []
         self._onUnregister = []
         self._onNRegistered = []
     
     #---------------------------------------------------------------------------
     # Methods used to save the engine info to a log file
     #---------------------------------------------------------------------------
     
     def _buildEngineInfoString(self, id, ip, port, pid):
         if id is None:
             id = -99
         if ip is None:
             ip = "-99"
         if port is None:
             port = -99
         if pid is None:
             pid = -99
         return "Engine Info: %d %s %d %d" % (id, ip , port, pid)
         
     def _logEngineInfo(self, id, ip, port, pid):
         log.msg(self._buildEngineInfoString(id,ip,port,pid))
     
     def _getEngineInfoLogFile(self):
         # Store all logs inside the ipython directory
-        ipdir = cutils.get_ipython_dir()
+        ipdir = get_ipython_dir()
         pjoin = os.path.join
         logdir_base = pjoin(ipdir,'log')
         if not os.path.isdir(logdir_base):
             os.makedirs(logdir_base)
         logfile = os.path.join(logdir_base,'ipcontroller-%s-engine-info.log' % os.getpid())
         return logfile
     
     def _logEngineInfoToFile(self, id, ip, port, pid):
         """Log info about an engine to a log file.
         
         When an engine registers with a ControllerService, the ControllerService
         saves information about the engine to a log file.  That information
         can be useful for various purposes, such as killing hung engines, etc.
         
         This method takes the assigned id, ip/port and pid of the engine
         and saves it to a file of the form:
         
         ~/.ipython/log/ipcontroller-###-engine-info.log
         
         where ### is the pid of the controller.
         
         Each line of this file has the form:
         
         Engine Info: ip ip port pid
         
         If any of the entries are not known, they are replaced by -99.
         """
         
         fname = self._getEngineInfoLogFile()
         f = open(fname, 'a')
         s = self._buildEngineInfoString(id,ip,port,pid)
         f.write(s + '\n')
         f.close()
     
     #---------------------------------------------------------------------------
     # IControllerCore methods
     #---------------------------------------------------------------------------
         
     def register_engine(self, remoteEngine, id=None,
         ip=None, port=None, pid=None):
         """Register new engine connection"""
         
         # What happens if these assertions fail?
         assert IEngineCore.providedBy(remoteEngine), \
             "engine passed to register_engine doesn't provide IEngineCore"
         assert IEngineSerialized.providedBy(remoteEngine), \
             "engine passed to register_engine doesn't provide IEngineSerialized"
         assert IEngineQueued.providedBy(remoteEngine), \
             "engine passed to register_engine doesn't provide IEngineQueued"
         assert isinstance(id, int) or id is None, \
             "id to register_engine must be an integer or None"
         assert isinstance(ip, str) or ip is None, \
             "ip to register_engine must be a string or None"
         assert isinstance(port, int) or port is None, \
             "port to register_engine must be an integer or None"
         assert isinstance(pid, int) or pid is None, \
             "pid to register_engine must be an integer or None"
             
         desiredID = id
         if desiredID in self.engines.keys():
             desiredID = None
             
         if desiredID in self.availableIDs:
             getID = desiredID
             self.availableIDs.remove(desiredID)
         else:
             getID = self.availableIDs.pop()
         remoteEngine.id = getID
         remoteEngine.service = self
         self.engines[getID] = remoteEngine
 
         # Log the Engine Information for monitoring purposes
         self._logEngineInfoToFile(getID, ip, port, pid)
 
         msg = "registered engine with id: %i" %getID
         log.msg(msg)
         
         for i in range(len(self._onRegister)):
             (f,args,kwargs,ifid) = self._onRegister[i]
             try:
                 if ifid:
                     f(getID, *args, **kwargs)
                 else:
                     f(*args, **kwargs)
             except:
                 self._onRegister.pop(i)
         
         # Call functions when the nth engine is registered and them remove them
         for i, (n, f, args, kwargs) in enumerate(self._onNRegistered):
             if len(self.engines.keys()) == n:
                 try:
                     try:
                         f(*args, **kwargs)
                     except:
                         log.msg("Function %r failed when the %ith engine registered" % (f, n))
                 finally:
                     self._onNRegistered.pop(i)
         
         return {'id':getID}
     
     def unregister_engine(self, id):
         """Unregister engine by id."""
         
         assert isinstance(id, int) or id is None, \
             "id to unregister_engine must be an integer or None"
         
         msg = "unregistered engine with id: %i" %id
         log.msg(msg)
         try:
             del self.engines[id]
         except KeyError:
             log.msg("engine with id %i was not registered" % id)
         else:
             if not self.saveIDs:
                 self.availableIDs.append(id)
                 # Sort to assign lower ids first
                 self.availableIDs.sort(reverse=True) 
             else:
                 log.msg("preserving id %i" %id)
             
             for i in range(len(self._onUnregister)):
                 (f,args,kwargs,ifid) = self._onUnregister[i]
                 try:
                     if ifid:
                         f(id, *args, **kwargs)
                     else:
                         f(*args, **kwargs)
                 except:
                     self._onUnregister.pop(i)
     
     def on_register_engine_do(self, f, includeID, *args, **kwargs):
         assert callable(f), "f must be callable"
         self._onRegister.append((f,args,kwargs,includeID))
 
     def on_unregister_engine_do(self, f, includeID, *args, **kwargs):
         assert callable(f), "f must be callable"
         self._onUnregister.append((f,args,kwargs,includeID))
     
     def on_register_engine_do_not(self, f):
         for i in range(len(self._onRegister)):
             g = self._onRegister[i][0]
             if f == g:
                 self._onRegister.pop(i)
                 return
     
     def on_unregister_engine_do_not(self, f):
         for i in range(len(self._onUnregister)):
             g = self._onUnregister[i][0]
             if f == g:
                 self._onUnregister.pop(i)
                 return
 
     def on_n_engines_registered_do(self, n, f, *args, **kwargs):
         if len(self.engines.keys()) >= n:
             f(*args, **kwargs)
         else:
             self._onNRegistered.append((n,f,args,kwargs))
             
 
 #-------------------------------------------------------------------------------
 # Base class for adapting controller to different client APIs
 #-------------------------------------------------------------------------------
 
 class ControllerAdapterBase(object):
     """All Controller adapters should inherit from this class.
     
     This class provides a wrapped version of the IControllerBase interface that
     can be used to easily create new custom controllers.  Subclasses of this
     will provide a full implementation of IControllerBase.
     
     This class doesn't implement any client notification mechanism.  That
     is up to subclasses.
     """
     
     implements(IControllerBase)
     
     def __init__(self, controller):
         self.controller = controller
         # Needed for IControllerCore
         self.engines = self.controller.engines
         
     def register_engine(self, remoteEngine, id=None,
         ip=None, port=None, pid=None):
         return self.controller.register_engine(remoteEngine, 
             id, ip, port, pid)
     
     def unregister_engine(self, id):
         return self.controller.unregister_engine(id)
 
     def on_register_engine_do(self, f, includeID, *args, **kwargs):
         return self.controller.on_register_engine_do(f, includeID, *args, **kwargs)
 
     def on_unregister_engine_do(self, f, includeID, *args, **kwargs):
         return self.controller.on_unregister_engine_do(f, includeID, *args, **kwargs)        
     
     def on_register_engine_do_not(self, f):
         return self.controller.on_register_engine_do_not(f)
     
     def on_unregister_engine_do_not(self, f):
         return self.controller.on_unregister_engine_do_not(f)        
 
     def on_n_engines_registered_do(self, n, f, *args, **kwargs):
         return self.controller.on_n_engines_registered_do(n, f, *args, **kwargs)

IPython/kernel/core/interpreter.py

@@ -1,747 +1,754 b''
 # encoding: utf-8
 
 """Central interpreter object for an IPython engine.
 
 The interpreter is the object whose job is to process lines of user input and
 actually execute them in the user's namespace.
 """
 
 __docformat__ = "restructuredtext en"
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
 
 # Standard library imports.
 from types import FunctionType
 
 import __builtin__
 import codeop
 import compiler
 import sys
 import traceback
 
 # Local imports.
 from IPython.kernel.core import ultraTB
 from IPython.kernel.core.display_trap import DisplayTrap
 from IPython.kernel.core.macro import Macro
 from IPython.kernel.core.prompts import CachedOutput
 from IPython.kernel.core.traceback_trap import TracebackTrap
 from IPython.kernel.core.util import Bunch, system_shell
 from IPython.external.Itpl import ItplNS
 
 # Global constants
 COMPILER_ERROR = 'error'
 INCOMPLETE_INPUT = 'incomplete'
 COMPLETE_INPUT = 'complete'
 
 ##############################################################################
 # TEMPORARY!!! fake configuration, while we decide whether to use tconfig or
 # not
 
 rc = Bunch()
 rc.cache_size = 100
 rc.pprint = True
 rc.separate_in = '\n'
 rc.separate_out = '\n'
 rc.separate_out2 = ''
 rc.prompt_in1 = r'In [\#]: '
 rc.prompt_in2 = r'   .\\D.: '
 rc.prompt_out = ''
 rc.prompts_pad_left = False
 
 ##############################################################################
 
 # Top-level utilities
 def default_display_formatters():
     """ Return a list of default display formatters.
     """
 
     from display_formatter import PPrintDisplayFormatter, ReprDisplayFormatter
     return [PPrintDisplayFormatter(), ReprDisplayFormatter()]
 
 def default_traceback_formatters():
     """ Return a list of default traceback formatters.
     """
 
     from traceback_formatter import PlainTracebackFormatter
     return [PlainTracebackFormatter()]
 
 # Top-level classes
 class NotDefined(object):    pass
 
 class Interpreter(object):
     """ An interpreter object.
 
     fixme: needs to negotiate available formatters with frontends.
 
     Important: the interpeter should be built so that it exposes a method
     for each attribute/method of its sub-object. This way it can be
     replaced by a network adapter.
     """
 
     def __init__(self, user_ns=None, global_ns=None,translator=None,
                  magic=None, display_formatters=None,
                  traceback_formatters=None, output_trap=None, history=None,
                  message_cache=None, filename='<string>', config=None):
 
         # The local/global namespaces for code execution
         local_ns = user_ns  # compatibility name
         if local_ns is None:
             local_ns = {}
         self.user_ns = local_ns
         # The local namespace
         if global_ns is None:
             global_ns = {}
         self.user_global_ns = global_ns
 
         # An object that will translate commands into executable Python.
         # The current translator does not work properly so for now we are going
         # without!
         # if translator is None:
         #             from IPython.kernel.core.translator import IPythonTranslator
         #             translator = IPythonTranslator()
         self.translator = translator
 
         # An object that maintains magic commands.
         if magic is None:
             from IPython.kernel.core.magic import Magic
             magic = Magic(self)
         self.magic = magic
 
         # A list of formatters for the displayhook.
         if display_formatters is None:
             display_formatters = default_display_formatters()
         self.display_formatters = display_formatters
 
         # A list of formatters for tracebacks.
         if traceback_formatters is None:
             traceback_formatters = default_traceback_formatters()
         self.traceback_formatters = traceback_formatters
 
         # The object trapping stdout/stderr.
         if output_trap is None:
             from IPython.kernel.core.output_trap import OutputTrap
             output_trap = OutputTrap()
         self.output_trap = output_trap
 
         # An object that manages the history.
         if history is None:
             from IPython.kernel.core.history import InterpreterHistory
             history = InterpreterHistory()
         self.history = history
         self.get_history_item = history.get_history_item
         self.get_history_input_cache = history.get_input_cache
         self.get_history_input_after = history.get_input_after
 
         # An object that caches all of the return messages.
         if message_cache is None:
             from IPython.kernel.core.message_cache import SimpleMessageCache
             message_cache = SimpleMessageCache()
         self.message_cache = message_cache
 
         # The "filename" of the code that is executed in this interpreter.
         self.filename = filename
 
         # An object that contains much configuration information.
         if config is None:
             # fixme: Move this constant elsewhere!
             config = Bunch(ESC_MAGIC='%')
         self.config = config
 
         # Hook managers.
         # fixme: make the display callbacks configurable. In the meantime,
         # enable macros.
         self.display_trap = DisplayTrap(
             formatters=self.display_formatters,
             callbacks=[self._possible_macro],
         )
         self.traceback_trap = TracebackTrap(
             formatters=self.traceback_formatters)
 
         # This is used temporarily for reformating exceptions in certain
         # cases.  It will go away once the ultraTB stuff is ported
         # to ipython1
         self.tbHandler = ultraTB.FormattedTB(color_scheme='NoColor',
                                                  mode='Context',
                                                  tb_offset=2)
 
         # An object that can compile commands and remember __future__
         # statements.
         self.command_compiler = codeop.CommandCompiler()
 
         # A replacement for the raw_input() and input() builtins. Change these
         # attributes later to configure them.
         self.raw_input_builtin = raw_input
         self.input_builtin = input
 
         # The number of the current cell.
         self.current_cell_number = 1
 
         # Initialize cache, set in/out prompts and printing system
         self.outputcache = CachedOutput(self,
                                         rc.cache_size,
                                         rc.pprint,
                                         input_sep = rc.separate_in,
                                         output_sep = rc.separate_out,
                                         output_sep2 = rc.separate_out2,
                                         ps1 = rc.prompt_in1,
                                         ps2 = rc.prompt_in2,
                                         ps_out = rc.prompt_out,
                                         pad_left = rc.prompts_pad_left)
 
         # Need to decide later if this is the right approach, but clients
         # commonly use sys.ps1/2, so it may be best to just set them here
         sys.ps1 = self.outputcache.prompt1.p_str
         sys.ps2 = self.outputcache.prompt2.p_str
 
         # This is the message dictionary assigned temporarily when running the
         # code.
         self.message = None
 
         self.setup_namespace()
 
 
     #### Public 'Interpreter' interface ########################################
 
     def formatTraceback(self, et, ev, tb, message=''):
         """Put a formatted version of the traceback into value and reraise.
         
         When exceptions have to be sent over the network, the traceback 
         needs to be put into the value of the exception in a nicely
         formatted way.  The method takes the type, value and tb of an 
         exception and puts a string representation of the tb into the 
         value of the exception and reraises it.
         
         Currently this method uses the ultraTb formatter from IPython trunk.
         Eventually it should simply use the traceback formatters in core
         that are loaded into self.tracback_trap.formatters.
         """
         tbinfo = self.tbHandler.text(et,ev,tb)
         ev._ipython_traceback_text = tbinfo
         return et, ev, tb
         
     def execute(self, commands, raiseException=True):
         """ Execute some IPython commands.
 
         1. Translate them into Python.
         2. Run them.
         3. Trap stdout/stderr.
         4. Trap sys.displayhook().
         5. Trap exceptions.
         6. Return a message object.
 
         Parameters
         ----------
         commands : str
             The raw commands that the user typed into the prompt.
         
         Returns
         -------
         message : dict
             The dictionary of responses. See the README.txt in this directory
             for an explanation of the format.
         """
 
         # Create a message dictionary with all of the information we will be
         # returning to the frontend and other listeners.
         message = self.setup_message()
 
         # Massage the input and store the raw and translated commands into
         # a dict.
         user_input = dict(raw=commands)
         if self.translator is not None:
             python = self.translator(commands, message)
             if python is None:
                 # Something went wrong with the translation. The translator
                 # should have added an appropriate entry to the message object.
                 return message
         else:
             python = commands
         user_input['translated'] = python
         message['input'] = user_input
 
         # Set the message object so that any magics executed in the code have
         # access.
         self.message = message
 
         # Set all of the output/exception traps.
         self.set_traps()
         
         # Actually execute the Python code.
         status = self.execute_python(python)
 
         # Unset all of the traps.
         self.unset_traps()
 
         # Unset the message object.
         self.message = None
 
         # Update the history variables in the namespace.
         # E.g. In, Out, _, __, ___
         if self.history is not None:
             self.history.update_history(self, python)
 
         # Let all of the traps contribute to the message and then clear their
         # stored information.
         self.output_trap.add_to_message(message)
         self.output_trap.clear()
         self.display_trap.add_to_message(message)
         self.display_trap.clear()
         self.traceback_trap.add_to_message(message)
         # Pull out the type, value and tb of the current exception
         # before clearing it.
         einfo = self.traceback_trap.args
         self.traceback_trap.clear()
         
         # Cache the message.
         self.message_cache.add_message(self.current_cell_number, message)
 
         # Bump the number.
         self.current_cell_number += 1
         
         # This conditional lets the execute method either raise any
         # exception that has occured in user code OR return the message
         # dict containing the traceback and other useful info.
         if raiseException and einfo:
             raise einfo[0],einfo[1],einfo[2]
         else:
             return message
 
     def generate_prompt(self, is_continuation):
         """Calculate and return a string with the prompt to display.
 
         :Parameters:
           is_continuation : bool
             Whether the input line is continuing multiline input or not, so
           that a proper continuation prompt can be computed."""
         
         if is_continuation:
             return str(self.outputcache.prompt2)
         else:
             return str(self.outputcache.prompt1)
 
     def execute_python(self, python):
         """ Actually run the Python code in the namespace.
 
         :Parameters:
 
         python : str
             Pure, exec'able Python code. Special IPython commands should have
             already been translated into pure Python.
         """
 
         # We use a CommandCompiler instance to compile the code so as to keep
         # track of __future__ imports.
         try:
             commands = self.split_commands(python)
         except (SyntaxError, IndentationError), e:
             # Save the exc_info so compilation related exceptions can be
             # reraised
             self.traceback_trap.args = sys.exc_info()
             self.pack_exception(self.message,e)
             return None
 
         for cmd in commands:
             try:
                 code = self.command_compiler(cmd, self.filename, 'single')
             except (SyntaxError, OverflowError, ValueError), e:
                 self.traceback_trap.args = sys.exc_info()
                 self.pack_exception(self.message,e)
                 # No point in continuing if one block raised
                 return None
             else:
                 self.execute_block(code)
 
     def execute_block(self,code):
         """Execute a single block of code in the user namespace.
 
         Return value: a flag indicating whether the code to be run completed
         successfully:
 
           - 0: successful execution.
           - 1: an error occurred.
           """
         
         outflag = 1 # start by assuming error, success will reset it
         try:
             exec code in self.user_ns
             outflag = 0
         except SystemExit:
             self.resetbuffer()
             self.traceback_trap.args = sys.exc_info()
         except:
             self.traceback_trap.args = sys.exc_info()
 
         return outflag
 
     def execute_macro(self, macro):
         """ Execute the value of a macro.
 
         Parameters
         ----------
         macro : Macro
         """
 
         python = macro.value
         if self.translator is not None:
             python = self.translator(python)
         self.execute_python(python)
 
     def getCommand(self, i=None):
         """Gets the ith message in the message_cache.
         
         This is implemented here for compatibility with the old ipython1 shell
         I am not sure we need this though.  I even seem to remember that we
         were going to get rid of it.
         """
         return self.message_cache.get_message(i)
 
     def reset(self):
         """Reset the interpreter.
         
         Currently this only resets the users variables in the namespace.
         In the future we might want to also reset the other stateful
         things like that the Interpreter has, like In, Out, etc.
         """
         self.user_ns.clear()
         self.setup_namespace()
 
     def complete(self,line,text=None, pos=None):
         """Complete the given text.
 
         :Parameters:
 
           text : str
             Text fragment to be completed on.  Typically this is
         """
         # fixme: implement
         raise NotImplementedError
 
     def push(self, ns):
         """ Put value into the namespace with name key.
         
         Parameters
         ----------
         **kwds
         """
 
         self.user_ns.update(ns)
 
     def push_function(self, ns):
         # First set the func_globals for all functions to self.user_ns
         new_kwds = {}
         for k, v in ns.iteritems():
             if not isinstance(v, FunctionType):
                 raise TypeError("function object expected")
             new_kwds[k] = FunctionType(v.func_code, self.user_ns)
         self.user_ns.update(new_kwds)        
 
     def pack_exception(self,message,exc):
         message['exception'] = exc.__class__
         message['exception_value'] = \
             traceback.format_exception_only(exc.__class__, exc)
 
     def feed_block(self, source, filename='<input>', symbol='single'):
         """Compile some source in the interpreter.
 
         One several things can happen:
 
         1) The input is incorrect; compile_command() raised an
         exception (SyntaxError or OverflowError).
 
         2) The input is incomplete, and more input is required;
         compile_command() returned None.  Nothing happens.
 
         3) The input is complete; compile_command() returned a code
         object.  The code is executed by calling self.runcode() (which
         also handles run-time exceptions, except for SystemExit).
 
         The return value is:
 
           - True in case 2
 
           - False in the other cases, unless an exception is raised, where
           None is returned instead.  This can be used by external callers to
           know whether to continue feeding input or not.
 
         The return value can be used to decide whether to use sys.ps1 or
         sys.ps2 to prompt the next line."""
 
         self.message = self.setup_message()
 
         try:
             code = self.command_compiler(source,filename,symbol)
         except (OverflowError, SyntaxError, IndentationError, ValueError ), e:
             # Case 1
             self.traceback_trap.args = sys.exc_info()
             self.pack_exception(self.message,e)
             return COMPILER_ERROR,False
 
         if code is None:
             # Case 2: incomplete input.  This means that the input can span
             # multiple lines.  But we still need to decide when to actually
             # stop taking user input.  Later we'll add auto-indentation support
             # somehow.  In the meantime, we'll just stop if there are two lines
             # of pure whitespace at the end.
             last_two = source.rsplit('\n',2)[-2:]
             print 'last two:',last_two  # dbg
             if len(last_two)==2 and all(s.isspace() for s in last_two):
                 return COMPLETE_INPUT,False
             else:
                 return INCOMPLETE_INPUT, True
         else:
             # Case 3
             return COMPLETE_INPUT, False
         
     def pull(self, keys):
         """ Get an item out of the namespace by key.
         
         Parameters
         ----------
         key : str
 
         Returns
         -------
         value : object
 
         Raises
         ------
         TypeError if the key is not a string.
         NameError if the object doesn't exist.
         """
 
         if isinstance(keys, str):
             result = self.user_ns.get(keys, NotDefined())
             if isinstance(result, NotDefined):
                 raise NameError('name %s is not defined' % keys)        
         elif isinstance(keys, (list, tuple)):
             result = []
             for key in keys:
                 if not isinstance(key, str):
                     raise TypeError("objects must be keyed by strings.")
                 else:
                     r = self.user_ns.get(key, NotDefined())
                     if isinstance(r, NotDefined):
                         raise NameError('name %s is not defined' % key)
                     else:
                         result.append(r)
                     if len(keys)==1:
                         result = result[0]
         else:
             raise TypeError("keys must be a strong or a list/tuple of strings")
         return result
 
     def pull_function(self, keys):
         return self.pull(keys)
 
     #### Interactive user API ##################################################
 
     def ipsystem(self, command):
         """ Execute a command in a system shell while expanding variables in the
         current namespace.
 
         Parameters
         ----------
         command : str
         """
 
         # Expand $variables.
         command = self.var_expand(command)
 
         system_shell(command,
             header='IPython system call: ',
             verbose=self.rc.system_verbose,
         )
 
     def ipmagic(self, arg_string):
         """ Call a magic function by name.
 
         ipmagic('name -opt foo bar') is equivalent to typing at the ipython
         prompt:
 
         In[1]: %name -opt foo bar
 
         To call a magic without arguments, simply use ipmagic('name').
 
         This provides a proper Python function to call IPython's magics in any
         valid Python code you can type at the interpreter, including loops and
         compound statements.  It is added by IPython to the Python builtin
         namespace upon initialization.
 
         Parameters
         ----------
         arg_string : str
             A string containing the name of the magic function to call and any
             additional arguments to be passed to the magic.
 
         Returns
         -------
         something : object
             The return value of the actual object.
         """
 
         # Taken from IPython.
         raise NotImplementedError('Not ported yet')
 
         args = arg_string.split(' ', 1)
         magic_name = args[0]
         magic_name = magic_name.lstrip(self.config.ESC_MAGIC)
 
         try:
             magic_args = args[1]
         except IndexError:
             magic_args = ''
         fn = getattr(self.magic, 'magic_'+magic_name, None)
         if fn is None:
             self.error("Magic function `%s` not found." % magic_name)
         else:
             magic_args = self.var_expand(magic_args)
             return fn(magic_args)
 
 
     #### Private 'Interpreter' interface #######################################
 
     def setup_message(self):
         """Return a message object.
 
         This method prepares and returns a message dictionary.  This dict
         contains the various fields that are used to transfer information about
         execution, results, tracebacks, etc, to clients (either in or out of
         process ones).  Because of the need to work with possibly out of
         process clients, this dict MUST contain strictly pickle-safe values.
         """
 
         return dict(number=self.current_cell_number)
     
     def setup_namespace(self):
         """ Add things to the namespace.
         """
 
         self.user_ns.setdefault('__name__', '__main__')
         self.user_ns.setdefault('__builtins__', __builtin__)
         self.user_ns['__IP'] = self
         if self.raw_input_builtin is not None:
             self.user_ns['raw_input'] = self.raw_input_builtin
         if self.input_builtin is not None:
             self.user_ns['input'] = self.input_builtin
 
         builtin_additions = dict(
             ipmagic=self.ipmagic,
         )
         __builtin__.__dict__.update(builtin_additions)
 
         if self.history is not None:
             self.history.setup_namespace(self.user_ns)
 
     def set_traps(self):
         """ Set all of the output, display, and traceback traps.
         """
 
         self.output_trap.set()
         self.display_trap.set()
         self.traceback_trap.set()
 
     def unset_traps(self):
         """ Unset all of the output, display, and traceback traps.
         """
 
         self.output_trap.unset()
         self.display_trap.unset()
         self.traceback_trap.unset()
 
     def split_commands(self, python):
         """ Split multiple lines of code into discrete commands that can be
         executed singly.
 
         Parameters
         ----------
         python : str
             Pure, exec'able Python code.
 
         Returns
         -------
         commands : list of str
             Separate commands that can be exec'ed independently.
         """
 
         # compiler.parse treats trailing spaces after a newline as a
         # SyntaxError.  This is different than codeop.CommandCompiler, which
         # will compile the trailng spaces just fine.  We simply strip any
         # trailing whitespace off.  Passing a string with trailing whitespace
         # to exec will fail however.  There seems to be some inconsistency in
         # how trailing whitespace is handled, but this seems to work.
         python = python.strip()
 
+        # The compiler module does not like unicode. We need to convert
+        # it encode it:
+        if isinstance(python, unicode):
+            # Use the utf-8-sig BOM so the compiler detects this a UTF-8
+            # encode string.
+            python = '\xef\xbb\xbf' + python.encode('utf-8')
+
         # The compiler module will parse the code into an abstract syntax tree.
         ast = compiler.parse(python)
 
         # Uncomment to help debug the ast tree
         # for n in ast.node:
         #     print n.lineno,'->',n
                 
         # Each separate command is available by iterating over ast.node. The
         # lineno attribute is the line number (1-indexed) beginning the commands
         # suite.
         # lines ending with ";" yield a Discard Node that doesn't have a lineno
         # attribute.  These nodes can and should be discarded.  But there are
         # other situations that cause Discard nodes that shouldn't be discarded.
         # We might eventually discover other cases where lineno is None and have
         # to put in a more sophisticated test.
         linenos = [x.lineno-1 for x in ast.node if x.lineno is not None]
 
         # When we finally get the slices, we will need to slice all the way to
         # the end even though we don't have a line number for it. Fortunately,
         # None does the job nicely.
         linenos.append(None)
         lines = python.splitlines()
 
         # Create a list of atomic commands.
         cmds = []
         for i, j in zip(linenos[:-1], linenos[1:]):
             cmd = lines[i:j]
             if cmd:
                 cmds.append('\n'.join(cmd)+'\n')
 
         return cmds
 
     def error(self, text):
         """ Pass an error message back to the shell.
 
         Preconditions
         -------------
         This should only be called when self.message is set. In other words,
         when code is being executed.
 
         Parameters
         ----------
         text : str
         """
 
         errors = self.message.get('IPYTHON_ERROR', [])
         errors.append(text)
 
     def var_expand(self, template):
         """ Expand $variables in the current namespace using Itpl.
 
         Parameters
         ----------
         template : str
         """
 
         return str(ItplNS(template, self.user_ns))
 
     def _possible_macro(self, obj):
         """ If the object is a macro, execute it.
         """
 
         if isinstance(obj, Macro):
             self.execute_macro(obj)
 

IPython/kernel/scripts/ipcluster.py

@@ -1,324 +1,347 b''
 #!/usr/bin/env python
 # encoding: utf-8
 
 """Start an IPython cluster conveniently, either locally or remotely.
 
 Basic usage
 -----------
 
 For local operation, the simplest mode of usage is:
 
   %prog -n N
 
 where N is the number of engines you want started.
 
 For remote operation, you must call it with a cluster description file:
 
   %prog -f clusterfile.py
 
 The cluster file is a normal Python script which gets run via execfile().  You
 can have arbitrary logic in it, but all that matters is that at the end of the
 execution, it declares the variables 'controller', 'engines', and optionally
 'sshx'.  See the accompanying examples for details on what these variables must
 contain.
 
 
 Notes
 -----
 
 WARNING: this code is still UNFINISHED and EXPERIMENTAL!  It is incomplete,
 some listed options are not really implemented, and all of its interfaces are
 subject to change.
 
 When operating over SSH for a remote cluster, this program relies on the
 existence of a particular script called 'sshx'.  This script must live in the
 target systems where you'll be running your controller and engines, and is
 needed to configure your PATH and PYTHONPATH variables for further execution of
 python code at the other end of an SSH connection.  The script can be as simple
 as:
 
 #!/bin/sh
 . $HOME/.bashrc
 "$@"
 
 which is the default one provided by IPython.  You can modify this or provide
 your own.  Since it's quite likely that for different clusters you may need
 this script to configure things differently or that it may live in different
 locations, its full path can be set in the same file where you define the
 cluster setup.  IPython's order of evaluation for this variable is the
 following:
 
   a) Internal default: 'sshx'.  This only works if it is in the default system
   path which SSH sets up in non-interactive mode.
 
   b) Environment variable: if $IPYTHON_SSHX is defined, this overrides the
   internal default.
 
   c) Variable 'sshx' in the cluster configuration file: finally, this will
   override the previous two values.
  
 This code is Unix-only, with precious little hope of any of this ever working
 under Windows, since we need SSH from the ground up, we background processes,
 etc.  Ports of this functionality to Windows are welcome.
 
 
 Call summary
 ------------
 
     %prog [options]
 """
 
 __docformat__ = "restructuredtext en"
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # Stdlib imports
 #-------------------------------------------------------------------------------
 
 import os
 import signal
 import sys
 import time
 
 from optparse import OptionParser
 from subprocess import Popen,call
 
 #---------------------------------------------------------------------------
 # IPython imports
 #---------------------------------------------------------------------------
 from IPython.tools import utils
-from IPython.config import cutils
+from IPython.genutils import get_ipython_dir
 
 #---------------------------------------------------------------------------
 # Normal code begins
 #---------------------------------------------------------------------------
 
 def parse_args():
     """Parse command line and return opts,args."""
 
     parser = OptionParser(usage=__doc__)
     newopt = parser.add_option  # shorthand
 
     newopt("--controller-port", type="int", dest="controllerport",
            help="the TCP port the controller is listening on")
 
     newopt("--controller-ip", type="string", dest="controllerip",
            help="the TCP ip address of the controller")
 
     newopt("-n", "--num", type="int", dest="n",default=2,
            help="the number of engines to start")
 
     newopt("--engine-port", type="int", dest="engineport",
            help="the TCP port the controller will listen on for engine "
            "connections")
     
     newopt("--engine-ip", type="string", dest="engineip",
            help="the TCP ip address the controller will listen on "
            "for engine connections")
 
     newopt("--mpi", type="string", dest="mpi",
            help="use mpi with package:  for instance --mpi=mpi4py")
     
     newopt("-l", "--logfile", type="string", dest="logfile",
            help="log file name")
 
     newopt('-f','--cluster-file',dest='clusterfile',
            help='file describing a remote cluster')
 
     return parser.parse_args()
 
 def numAlive(controller,engines):
     """Return the number of processes still alive."""
     retcodes = [controller.poll()] + \
                [e.poll() for e in engines]
     return retcodes.count(None)
 
 stop = lambda pid: os.kill(pid,signal.SIGINT)
 kill = lambda pid: os.kill(pid,signal.SIGTERM)
 
 def cleanup(clean,controller,engines):
     """Stop the controller and engines with the given cleanup method."""
     
     for e in engines:
         if e.poll() is None:
             print 'Stopping engine, pid',e.pid
             clean(e.pid)
     if controller.poll() is None:
         print 'Stopping controller, pid',controller.pid
         clean(controller.pid)
 
 
 def ensureDir(path):
     """Ensure a directory exists or raise an exception."""
     if not os.path.isdir(path):
         os.makedirs(path)
 
 
 def startMsg(control_host,control_port=10105):
     """Print a startup message"""
     print
     print 'Your cluster is up and running.'
     print
     print 'For interactive use, you can make a MultiEngineClient with:'
     print
     print 'from IPython.kernel import client'
     print "mec = client.MultiEngineClient()"
     print
     print 'You can then cleanly stop the cluster from IPython using:'
     print
     print 'mec.kill(controller=True)'
     print
 
     
 def clusterLocal(opt,arg):
     """Start a cluster on the local machine."""
     
     # Store all logs inside the ipython directory
-    ipdir = cutils.get_ipython_dir()
+    ipdir = get_ipython_dir()
     pjoin = os.path.join
 
     logfile = opt.logfile
     if logfile is None:
         logdir_base = pjoin(ipdir,'log')
         ensureDir(logdir_base)
         logfile = pjoin(logdir_base,'ipcluster-')
 
     print 'Starting controller:',
     controller = Popen(['ipcontroller','--logfile',logfile,'-x','-y'])
     print 'Controller PID:',controller.pid
 
     print 'Starting engines:   ',
     time.sleep(5)
 
     englogfile = '%s%s-' % (logfile,controller.pid)
     mpi = opt.mpi
     if mpi: # start with mpi - killing the engines with sigterm will not work if you do this
         engines = [Popen(['mpirun', '-np', str(opt.n), 'ipengine', '--mpi', 
             mpi, '--logfile',englogfile])]
         # engines = [Popen(['mpirun', '-np', str(opt.n), 'ipengine', '--mpi', mpi])]
     else: # do what we would normally do
         engines = [ Popen(['ipengine','--logfile',englogfile])
                     for i in range(opt.n) ]
     eids = [e.pid for e in engines]
     print 'Engines PIDs:  ',eids
     print 'Log files: %s*' % englogfile
     
     proc_ids = eids + [controller.pid]
     procs = engines + [controller]
 
     grpid = os.getpgrp()
     try:
         startMsg('127.0.0.1')
         print 'You can also hit Ctrl-C to stop it, or use from the cmd line:'
         print
         print 'kill -INT',grpid
         print
         try:
             while True:
                 time.sleep(5)
         except:
             pass
     finally:
         print 'Stopping cluster.  Cleaning up...'
         cleanup(stop,controller,engines)
         for i in range(4):
             time.sleep(i+2)
             nZombies = numAlive(controller,engines)
             if  nZombies== 0:
                 print 'OK: All processes cleaned up.'
                 break
             print 'Trying again, %d processes did not stop...' % nZombies
             cleanup(kill,controller,engines)
             if numAlive(controller,engines) == 0:
                 print 'OK: All processes cleaned up.'
                 break
         else:
             print '*'*75
             print 'ERROR: could not kill some processes, try to do it',
             print 'manually.'
             zombies = []
             if controller.returncode is None:
                 print 'Controller is alive: pid =',controller.pid
                 zombies.append(controller.pid)
             liveEngines = [ e for e in engines if e.returncode is None ]
             for e in liveEngines:
                 print 'Engine is alive:     pid =',e.pid
                 zombies.append(e.pid)
             print
             print 'Zombie summary:',' '.join(map(str,zombies))
 
 def clusterRemote(opt,arg):
     """Start a remote cluster over SSH"""
 
+    # B. Granger, 9/3/08
+    # The launching of a remote cluster using SSH and a clusterfile
+    # is broken.  Because it won't be fixed before the 0.9 release, 
+    # we are removing it.  For now, we just print a message to the 
+    # user and abort.
+    
+    print """The launching of a remote IPython cluster using SSL
+and a clusterfile has been removed in this release.  
+It has been broken for a while and we are in the process 
+of building a new process management system that will be 
+used to provide a more robust way of starting an IPython
+cluster.
+
+For now remote clusters have to be launched using ipcontroller
+and ipengine separately.
+    """
+    sys.exit(1)
+
     # Load the remote cluster configuration
     clConfig = {}
     execfile(opt.clusterfile,clConfig)
     contConfig = clConfig['controller']
     engConfig = clConfig['engines']
     # Determine where to find sshx:
     sshx = clConfig.get('sshx',os.environ.get('IPYTHON_SSHX','sshx'))
     
     # Store all logs inside the ipython directory
-    ipdir = cutils.get_ipython_dir()
+    ipdir = get_ipython_dir()
     pjoin = os.path.join
 
     logfile = opt.logfile
     if logfile is None:
         logdir_base = pjoin(ipdir,'log')
         ensureDir(logdir_base)
         logfile = pjoin(logdir_base,'ipcluster')
 
     # Append this script's PID to the logfile name always
     logfile = '%s-%s' % (logfile,os.getpid())
     
     print 'Starting controller:'
     # Controller data:
     xsys = os.system
 
     contHost = contConfig['host']
     contLog = '%s-con-%s-' % (logfile,contHost)
     cmd = "ssh %s '%s' 'ipcontroller --logfile %s' &" % \
           (contHost,sshx,contLog)
     #print 'cmd:<%s>' % cmd  # dbg
     xsys(cmd)
     time.sleep(2)
 
     print 'Starting engines:   '
     for engineHost,engineData in engConfig.iteritems():
         if isinstance(engineData,int):
             numEngines = engineData
         else:
             raise NotImplementedError('port configuration not finished for engines')
 
         print 'Sarting %d engines on %s' % (numEngines,engineHost)
         engLog = '%s-eng-%s-' % (logfile,engineHost)
         for i in range(numEngines):
             cmd = "ssh %s '%s' 'ipengine --controller-ip %s --logfile %s' &" % \
                       (engineHost,sshx,contHost,engLog)
             #print 'cmd:<%s>' % cmd  # dbg
             xsys(cmd)
         # Wait after each host a little bit
         time.sleep(1)
             
     startMsg(contConfig['host'])
         
 def main():
     """Main driver for the two big options: local or remote cluster."""
     
+    if sys.platform=='win32':
+        print """ipcluster does not work on Microsoft Windows.  Please start
+your IPython cluster using the ipcontroller and ipengine scripts."""
+        sys.exit(1)
+    
     opt,arg = parse_args()
 
     clusterfile = opt.clusterfile
     if clusterfile:
         clusterRemote(opt,arg)
     else:
         clusterLocal(opt,arg)
         
             
 if __name__=='__main__':
     main()

IPython/kernel/scripts/ipengine.py

@@ -1,171 +1,172 b''
 #!/usr/bin/env python
 # encoding: utf-8
 
 """Start the IPython Engine."""
 
 __docformat__ = "restructuredtext en"
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
 
 # Python looks for an empty string at the beginning of sys.path to enable
 # importing from the cwd.
 import sys
 sys.path.insert(0, '')
 
 import sys, os
 from optparse import OptionParser
 
 from twisted.application import service
 from twisted.internet import reactor
 from twisted.python import log
 
 from IPython.kernel.fcutil import Tub, UnauthenticatedTub
 
 from IPython.kernel.core.config import config_manager as core_config_manager
 from IPython.config.cutils import import_item
 from IPython.kernel.engineservice import EngineService
 from IPython.kernel.config import config_manager as kernel_config_manager
 from IPython.kernel.engineconnector import EngineConnector
 
 
 #-------------------------------------------------------------------------------
 # Code
 #-------------------------------------------------------------------------------
 
 def start_engine():
     """
     Start the engine, by creating it and starting the Twisted reactor.
     
     This method does:
     
         * If it exists, runs the `mpi_import_statement` to call `MPI_Init`
         * Starts the engine logging
         * Creates an IPython shell and wraps it in an `EngineService`
         * Creates a `foolscap.Tub` to use in connecting to a controller.
         * Uses the tub and the `EngineService` along with a Foolscap URL
           (or FURL) to connect to the controller and register the engine
           with the controller
     """
     kernel_config = kernel_config_manager.get_config_obj()
     core_config = core_config_manager.get_config_obj()
     
 
     # Execute the mpi import statement that needs to call MPI_Init
     global mpi
     mpikey = kernel_config['mpi']['default']
     mpi_import_statement = kernel_config['mpi'].get(mpikey, None)
     if mpi_import_statement is not None:
         try:
             exec mpi_import_statement in globals()
         except:
             mpi = None
     else:
         mpi = None
     
     # Start logging
     logfile = kernel_config['engine']['logfile']
     if logfile:
         logfile = logfile + str(os.getpid()) + '.log'
         try:
             openLogFile = open(logfile, 'w')
         except:
             openLogFile = sys.stdout
     else:
         openLogFile = sys.stdout
     log.startLogging(openLogFile)
     
     # Create the underlying shell class and EngineService
     shell_class = import_item(core_config['shell']['shell_class'])
     engine_service = EngineService(shell_class, mpi=mpi)
     shell_import_statement = core_config['shell']['import_statement']
     if shell_import_statement:
         try:
             engine_service.execute(shell_import_statement)
         except:
             log.msg("Error running import_statement: %s" % sis)
     
     # Create the service hierarchy
     main_service = service.MultiService()
     engine_service.setServiceParent(main_service)
     tub_service = Tub()
     tub_service.setServiceParent(main_service)
     # This needs to be called before the connection is initiated
     main_service.startService()
     
     # This initiates the connection to the controller and calls
     # register_engine to tell the controller we are ready to do work
     engine_connector = EngineConnector(tub_service)
     furl_file = kernel_config['engine']['furl_file']
+    log.msg("Using furl file: %s" % furl_file)
     d = engine_connector.connect_to_controller(engine_service, furl_file)
     d.addErrback(lambda _: reactor.stop())
     
     reactor.run()
 
 
 def init_config():
     """
     Initialize the configuration using default and command line options.
     """
     
     parser = OptionParser()
     
     parser.add_option(
         "--furl-file",
         type="string", 
         dest="furl_file",
         help="The filename containing the FURL of the controller"
     )
     parser.add_option(
         "--mpi",
         type="string",
         dest="mpi",
         help="How to enable MPI (mpi4py, pytrilinos, or empty string to disable)"
     )
     parser.add_option(
         "-l",
         "--logfile",
         type="string",
         dest="logfile",
         help="log file name (default is stdout)"
     )
     parser.add_option(
         "--ipythondir",
         type="string",
         dest="ipythondir",
         help="look for config files and profiles in this directory"
     )
     
     (options, args) = parser.parse_args()
     
     kernel_config_manager.update_config_obj_from_default_file(options.ipythondir)
     core_config_manager.update_config_obj_from_default_file(options.ipythondir)
     
     kernel_config = kernel_config_manager.get_config_obj()
     # Now override with command line options
     if options.furl_file is not None:
         kernel_config['engine']['furl_file'] = options.furl_file
     if options.logfile is not None:
         kernel_config['engine']['logfile'] = options.logfile
     if options.mpi is not None:
         kernel_config['mpi']['default'] = options.mpi
 
 
 def main():
     """
     After creating the configuration information, start the engine.
     """
     init_config()
     start_engine()
 
 
 if __name__ == "__main__":
     main()
  No newline at end of file

IPython/kernel/tests/engineservicetest.py

@@ -1,373 +1,373 b''
 # encoding: utf-8
 
 """Test template for complete engine object"""
 
 __docformat__ = "restructuredtext en"
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
 
 import cPickle as pickle
 
 from twisted.internet import defer, reactor
 from twisted.python import failure
 from twisted.application import service
 import zope.interface as zi
 
 from IPython.kernel import newserialized
 from IPython.kernel import error
 from IPython.kernel.pickleutil import can, uncan
 import IPython.kernel.engineservice as es
 from IPython.kernel.core.interpreter import Interpreter
 from IPython.testing.parametric import Parametric, parametric
 
 #-------------------------------------------------------------------------------
 # Tests
 #-------------------------------------------------------------------------------
 
     
 # A sequence of valid commands run through execute
 validCommands = ['a=5',
                  'b=10',
                  'a=5; b=10; c=a+b',
                  'import math; 2.0*math.pi',
                  """def f():
     result = 0.0
     for i in range(10):
         result += i
 """,
                  'if 1<2: a=5',
                  """import time
 time.sleep(0.1)""",
                 """from math import cos;
 x = 1.0*cos(0.5)""", # Semicolons lead to Discard ast nodes that should be discarded
                 """from sets import Set
 s = Set()
    """, # Trailing whitespace should be allowed.
                 """import math
 math.cos(1.0)""", # Test a method call with a discarded return value
                 """x=1.0234
 a=5; b=10""", # Test an embedded semicolon
                 """x=1.0234
 a=5; b=10;""" # Test both an embedded and trailing semicolon
                  ]
                  
 # A sequence of commands that raise various exceptions
 invalidCommands = [('a=1/0',ZeroDivisionError),
                    ('print v',NameError),
                    ('l=[];l[0]',IndexError),
                    ("d={};d['a']",KeyError),
                    ("assert 1==0",AssertionError),
                    ("import abababsdbfsbaljasdlja",ImportError),
                    ("raise Exception()",Exception)]
 
 def testf(x):
     return 2.0*x
 
 globala = 99
 
 def testg(x):
     return  globala*x
 
 class IEngineCoreTestCase(object):
     """Test an IEngineCore implementer."""
 
     def createShell(self):
         return Interpreter()
 
     def catchQueueCleared(self, f):
         try:
             f.raiseException()
         except error.QueueCleared:
             pass
         
     def testIEngineCoreInterface(self):
         """Does self.engine claim to implement IEngineCore?"""
         self.assert_(es.IEngineCore.providedBy(self.engine))
         
     def testIEngineCoreInterfaceMethods(self):
         """Does self.engine have the methods and attributes in IEngineCore."""
         for m in list(es.IEngineCore):
             self.assert_(hasattr(self.engine, m))
     
     def testIEngineCoreDeferreds(self):
         d = self.engine.execute('a=5')
         d.addCallback(lambda _: self.engine.pull('a'))
         d.addCallback(lambda _: self.engine.get_result())
         d.addCallback(lambda _: self.engine.keys())
         d.addCallback(lambda _: self.engine.push(dict(a=10)))
         return d
     
     def runTestExecute(self, cmd):
         self.shell = Interpreter()
         actual = self.shell.execute(cmd)
         def compare(computed):
             actual['id'] = computed['id']
             self.assertEquals(actual, computed)
         d = self.engine.execute(cmd)
         d.addCallback(compare)
         return d
     
     @parametric
     def testExecute(cls):
         return [(cls.runTestExecute, cmd) for cmd in validCommands]
 
     def runTestExecuteFailures(self, cmd, exc):
         def compare(f):
             self.assertRaises(exc, f.raiseException)
         d = self.engine.execute(cmd)
         d.addErrback(compare)
         return d
     
     @parametric
     def testExecuteFailuresEngineService(cls):
         return [(cls.runTestExecuteFailures, cmd, exc)
                 for cmd, exc in invalidCommands]
     
     def runTestPushPull(self, o):
         d = self.engine.push(dict(a=o))
         d.addCallback(lambda r: self.engine.pull('a'))
         d.addCallback(lambda r: self.assertEquals(o,r))
         return d
     
     @parametric
     def testPushPull(cls):
         objs = [10,"hi there",1.2342354,{"p":(1,2)},None]
         return [(cls.runTestPushPull, o) for o in objs]
         
     def testPullNameError(self):
         d = self.engine.push(dict(a=5))
         d.addCallback(lambda _:self.engine.reset())
         d.addCallback(lambda _: self.engine.pull("a"))
         d.addErrback(lambda f: self.assertRaises(NameError, f.raiseException))
         return d        
     
     def testPushPullFailures(self):
         d = self.engine.pull('a')
         d.addErrback(lambda f: self.assertRaises(NameError, f.raiseException))
         d.addCallback(lambda _: self.engine.execute('l = lambda x: x'))
         d.addCallback(lambda _: self.engine.pull('l'))
         d.addErrback(lambda f: self.assertRaises(pickle.PicklingError, f.raiseException))
         d.addCallback(lambda _: self.engine.push(dict(l=lambda x: x)))
         d.addErrback(lambda f: self.assertRaises(pickle.PicklingError, f.raiseException))
         return d
         
     def testPushPullArray(self):
         try:
             import numpy
         except:
             return
         a = numpy.random.random(1000)
         d = self.engine.push(dict(a=a))
         d.addCallback(lambda _: self.engine.pull('a'))
         d.addCallback(lambda b: b==a)
         d.addCallback(lambda c: c.all())
         return self.assertDeferredEquals(d, True)
         
     def testPushFunction(self):
                     
         d = self.engine.push_function(dict(f=testf))
         d.addCallback(lambda _: self.engine.execute('result = f(10)'))
         d.addCallback(lambda _: self.engine.pull('result'))
         d.addCallback(lambda r: self.assertEquals(r, testf(10)))
         return d
 
     def testPullFunction(self):
         d = self.engine.push_function(dict(f=testf, g=testg))
         d.addCallback(lambda _: self.engine.pull_function(('f','g')))
         d.addCallback(lambda r: self.assertEquals(r[0](10), testf(10)))
         return d
         
     def testPushFunctionGlobal(self):
         """Make sure that pushed functions pick up the user's namespace for globals."""
         d = self.engine.push(dict(globala=globala))
         d.addCallback(lambda _: self.engine.push_function(dict(g=testg)))
         d.addCallback(lambda _: self.engine.execute('result = g(10)'))
         d.addCallback(lambda _: self.engine.pull('result'))
         d.addCallback(lambda r: self.assertEquals(r, testg(10)))
         return d
         
     def testGetResultFailure(self):
         d = self.engine.get_result(None)
         d.addErrback(lambda f: self.assertRaises(IndexError, f.raiseException))
         d.addCallback(lambda _: self.engine.get_result(10))
         d.addErrback(lambda f: self.assertRaises(IndexError, f.raiseException))
         return d
 
     def runTestGetResult(self, cmd):
         self.shell = Interpreter()
         actual = self.shell.execute(cmd)
         def compare(computed):
             actual['id'] = computed['id']
             self.assertEquals(actual, computed)
         d = self.engine.execute(cmd)
         d.addCallback(lambda r: self.engine.get_result(r['number']))
         d.addCallback(compare)
         return d
     
     @parametric
     def testGetResult(cls):
         return [(cls.runTestGetResult, cmd) for cmd in validCommands]
     
     def testGetResultDefault(self):
         cmd = 'a=5'
         shell = self.createShell()
         shellResult = shell.execute(cmd)
         def popit(dikt, key):
             dikt.pop(key)
             return dikt
         d = self.engine.execute(cmd)
         d.addCallback(lambda _: self.engine.get_result())
         d.addCallback(lambda r: self.assertEquals(shellResult, popit(r,'id')))
         return d
 
     def testKeys(self):
         d = self.engine.keys()
         d.addCallback(lambda s: isinstance(s, list))
         d.addCallback(lambda r: self.assertEquals(r, True))
         return d
             
 Parametric(IEngineCoreTestCase)
             
 class IEngineSerializedTestCase(object):
     """Test an IEngineCore implementer."""
         
     def testIEngineSerializedInterface(self):
         """Does self.engine claim to implement IEngineCore?"""
         self.assert_(es.IEngineSerialized.providedBy(self.engine))
         
     def testIEngineSerializedInterfaceMethods(self):
-        """Does self.engine have the methods and attributes in IEngireCore."""
+        """Does self.engine have the methods and attributes in IEngineCore."""
         for m in list(es.IEngineSerialized):
             self.assert_(hasattr(self.engine, m))
        
     def testIEngineSerializedDeferreds(self):
         dList = []
         d = self.engine.push_serialized(dict(key=newserialized.serialize(12345)))
         self.assert_(isinstance(d, defer.Deferred))
         dList.append(d)
         d = self.engine.pull_serialized('key')
         self.assert_(isinstance(d, defer.Deferred))
         dList.append(d)
         D = defer.DeferredList(dList)
         return D
                 
     def testPushPullSerialized(self):
         objs = [10,"hi there",1.2342354,{"p":(1,2)}]
         d = defer.succeed(None)
         for o in objs:
             self.engine.push_serialized(dict(key=newserialized.serialize(o)))
             value = self.engine.pull_serialized('key')
             value.addCallback(lambda serial: newserialized.IUnSerialized(serial).getObject())
             d = self.assertDeferredEquals(value,o,d)
         return d
 
     def testPullSerializedFailures(self):
         d = self.engine.pull_serialized('a')
         d.addErrback(lambda f: self.assertRaises(NameError, f.raiseException))
         d.addCallback(lambda _: self.engine.execute('l = lambda x: x'))
         d.addCallback(lambda _: self.engine.pull_serialized('l'))
         d.addErrback(lambda f: self.assertRaises(pickle.PicklingError, f.raiseException))
         return d
 
 Parametric(IEngineSerializedTestCase)
 
 class IEngineQueuedTestCase(object):
     """Test an IEngineQueued implementer."""
         
     def testIEngineQueuedInterface(self):
         """Does self.engine claim to implement IEngineQueued?"""
         self.assert_(es.IEngineQueued.providedBy(self.engine))
         
     def testIEngineQueuedInterfaceMethods(self):
-        """Does self.engine have the methods and attributes in IEngireQueued."""
+        """Does self.engine have the methods and attributes in IEngineQueued."""
         for m in list(es.IEngineQueued):
             self.assert_(hasattr(self.engine, m))
             
     def testIEngineQueuedDeferreds(self): 
         dList = []
         d = self.engine.clear_queue()
         self.assert_(isinstance(d, defer.Deferred))
         dList.append(d)
         d = self.engine.queue_status()
         self.assert_(isinstance(d, defer.Deferred))
         dList.append(d)
         D = defer.DeferredList(dList)
         return D
             
     def testClearQueue(self):
         result = self.engine.clear_queue()
         d1 = self.assertDeferredEquals(result, None)
         d1.addCallback(lambda _: self.engine.queue_status())
         d2 = self.assertDeferredEquals(d1, {'queue':[], 'pending':'None'})
         return d2
         
     def testQueueStatus(self):
         result = self.engine.queue_status()
         result.addCallback(lambda r: 'queue' in r and 'pending' in r)
         d = self.assertDeferredEquals(result, True)
         return d
 
 Parametric(IEngineQueuedTestCase)
 
 class IEnginePropertiesTestCase(object):
     """Test an IEngineProperties implementor."""
     
     def testIEnginePropertiesInterface(self):
         """Does self.engine claim to implement IEngineProperties?"""
         self.assert_(es.IEngineProperties.providedBy(self.engine))
     
     def testIEnginePropertiesInterfaceMethods(self):
-        """Does self.engine have the methods and attributes in IEngireProperties."""
+        """Does self.engine have the methods and attributes in IEngineProperties."""
         for m in list(es.IEngineProperties):
             self.assert_(hasattr(self.engine, m))
     
     def testGetSetProperties(self):
         dikt = dict(a=5, b='asdf', c=True, d=None, e=range(5))
         d = self.engine.set_properties(dikt)
         d.addCallback(lambda r: self.engine.get_properties())
         d = self.assertDeferredEquals(d, dikt)
         d.addCallback(lambda r: self.engine.get_properties(('c',)))
         d = self.assertDeferredEquals(d, {'c': dikt['c']})
         d.addCallback(lambda r: self.engine.set_properties(dict(c=False)))
         d.addCallback(lambda r: self.engine.get_properties(('c', 'd')))
         d = self.assertDeferredEquals(d, dict(c=False, d=None))
         return d
     
     def testClearProperties(self):
         dikt = dict(a=5, b='asdf', c=True, d=None, e=range(5))
         d = self.engine.set_properties(dikt)
         d.addCallback(lambda r: self.engine.clear_properties())
         d.addCallback(lambda r: self.engine.get_properties())
         d = self.assertDeferredEquals(d, {})
         return d
     
     def testDelHasProperties(self):
         dikt = dict(a=5, b='asdf', c=True, d=None, e=range(5))
         d = self.engine.set_properties(dikt)
         d.addCallback(lambda r: self.engine.del_properties(('b','e')))
         d.addCallback(lambda r: self.engine.has_properties(('a','b','c','d','e')))
         d = self.assertDeferredEquals(d, [True, False, True, True, False])
         return d
     
     def testStrictDict(self):
         s = """from IPython.kernel.engineservice import get_engine
 p = get_engine(%s).properties"""%self.engine.id
         d = self.engine.execute(s)
         d.addCallback(lambda r: self.engine.execute("p['a'] = lambda _:None"))
         d = self.assertDeferredRaises(d, error.InvalidProperty)
         d.addCallback(lambda r: self.engine.execute("p['a'] = range(5)"))
         d.addCallback(lambda r: self.engine.execute("p['a'].append(5)"))
         d.addCallback(lambda r: self.engine.get_properties('a'))
         d = self.assertDeferredEquals(d, dict(a=range(5)))
         return d
         
 Parametric(IEnginePropertiesTestCase)

IPython/testing/plugin/Makefile

@@ -1,75 +1,74 b''
 # Set this prefix to where you want to install the plugin
-PREFIX=~/usr/local
-PREFIX=~/tmp/local
+PREFIX=/usr/local
 
 NOSE0=nosetests -vs --with-doctest --doctest-tests --detailed-errors
 NOSE=nosetests -vvs --with-ipdoctest --doctest-tests --doctest-extension=txt \
 --detailed-errors
 
 SRC=ipdoctest.py setup.py ../decorators.py
 
 # Default target for clean 'make'
 default: iplib
 
 # The actual plugin installation
 plugin: IPython_doctest_plugin.egg-info
 
 # Simple targets that test one thing
 simple: plugin simple.py
 	$(NOSE) simple.py
 
 dtest: plugin dtexample.py
 	$(NOSE) dtexample.py
 
 rtest: plugin test_refs.py
 	$(NOSE) test_refs.py
 
 test: plugin dtexample.py
 	$(NOSE) dtexample.py test*.py test*.txt
 
 deb: plugin dtexample.py
 	$(NOSE) test_combo.txt
 
 # IPython tests
 deco:
 	$(NOSE0) IPython.testing.decorators
 
 magic: plugin
 	$(NOSE) IPython.Magic
 
 ipipe: plugin
 	$(NOSE) IPython.Extensions.ipipe
 
 iplib: plugin
 	$(NOSE)  IPython.iplib
 
 strd: plugin
 	$(NOSE) IPython.strdispatch
 
 engine: plugin
 	$(NOSE) IPython.kernel
 
 tf: plugin
 	$(NOSE) IPython.config.traitlets
 
 # All of ipython itself
 ipython: plugin
 	$(NOSE) IPython
 
 
 # Combined targets
 sr: rtest strd
 
 base: dtest rtest test strd deco
 
 quick: base iplib ipipe
 
 all: base ipython
 
 # Main plugin and cleanup
 IPython_doctest_plugin.egg-info: $(SRC)
 	python setup.py install --prefix=$(PREFIX)
 	touch $@
 
 clean:
 	rm -rf IPython_doctest_plugin.egg-info *~ *pyc build/ dist/

docs/examples/kernel/mcpricer.py

@@ -1,42 +1,43 b''
 import numpy as N
 from math import *
 
 class MCOptionPricer(object):
     def __init__(self, S=100.0, K=100.0, sigma=0.25, r=0.05, days=260, paths=10000):
         self.S = S
         self.K = K
         self.sigma = sigma
         self.r = r
         self.days = days
         self.paths = paths
         self.h = 1.0/self.days
         self.const1 = exp((self.r-0.5*self.sigma**2)*self.h)
         self.const2 = self.sigma*sqrt(self.h)
         
     def run(self):
         stock_price = self.S*N.ones(self.paths, dtype='float64')
         stock_price_sum = N.zeros(self.paths, dtype='float64')
         for j in range(self.days):
             growth_factor = self.const1*N.exp(self.const2*N.random.standard_normal(self.paths))
             stock_price = stock_price*growth_factor
             stock_price_sum = stock_price_sum + stock_price
         stock_price_avg = stock_price_sum/self.days
         zeros = N.zeros(self.paths, dtype='float64')
         r_factor = exp(-self.r*self.h*self.days)
         self.vanilla_put = r_factor*N.mean(N.maximum(zeros,self.K-stock_price))
         self.asian_put = r_factor*N.mean(N.maximum(zeros,self.K-stock_price_avg))
         self.vanilla_call = r_factor*N.mean(N.maximum(zeros,stock_price-self.K))
         self.asian_call = r_factor*N.mean(N.maximum(zeros,stock_price_avg-self.K))
 
 
 def main():
     op = MCOptionPricer()
     op.run()
     print "Vanilla Put Price = ", op.vanilla_put
     print "Asian Put Price = ", op.asian_put
     print "Vanilla Call Price = ", op.vanilla_call
     print "Asian Call Price = ", op.asian_call
 
 
 if __name__ == '__main__':
     main()
+

docs/examples/kernel/rmt.ipy

@@ -1,57 +1,57 b''
 #-------------------------------------------------------------------------------
 # Driver code that the client runs.
 #-------------------------------------------------------------------------------
 # To run this code start a controller and engines using:
 # ipcluster -n 2
 # Then run the scripts by doing irunner rmt.ipy or by starting ipython and
 # doing run rmt.ipy.
 
 from rmtkernel import *
 from IPython.kernel import client
 
 
 def wignerDistribution(s):
     """Returns (s, rho(s)) for the Wigner GOE distribution."""
     return (numpy.pi*s/2.0) * numpy.exp(-numpy.pi*s**2/4.)
 
 
 def generateWignerData():
     s = numpy.linspace(0.0,4.0,400)
     rhos = wignerDistribution(s)
     return s, rhos
     
 
 def serialDiffs(num, N):
     diffs = ensembleDiffs(num, N)
     normalizedDiffs = normalizeDiffs(diffs)
     return normalizedDiffs
 
 
 def parallelDiffs(rc, num, N):
     nengines = len(rc.get_ids())
     num_per_engine = num/nengines
     print "Running with", num_per_engine, "per engine."
     rc.push(dict(num_per_engine=num_per_engine, N=N))
     rc.execute('diffs = ensembleDiffs(num_per_engine, N)')
 	# gather blocks always for now
     pr = rc.gather('diffs')
     return pr.r
 
 
 # Main code
 if __name__ == '__main__':
     rc = client.MultiEngineClient()
     print "Distributing code to engines..."
     r = rc.run('rmtkernel.py')
     rc.block = False
 
     # Simulation parameters
     nmats = 100
     matsize = 30
     
     %timeit -n1 -r1 serialDiffs(nmats,matsize)
     %timeit -n1 -r1 parallelDiffs(rc, nmats, matsize)
 
     # Uncomment these to plot the histogram
-    import pylab
-    pylab.hist(parallelDiffs(rc,matsize,matsize))
+    # import pylab
+    # pylab.hist(parallelDiffs(rc,matsize,matsize))

scripts/ipython_win_post_install.py

@@ -1,89 +1,106 b''
 #!python
 """Windows-specific part of the installation"""
 
 import os, sys, shutil
+pjoin = os.path.join
 
 def mkshortcut(target,description,link_file,*args,**kw):
     """make a shortcut if it doesn't exist, and register its creation"""
     
     create_shortcut(target, description, link_file,*args,**kw)
     file_created(link_file)
 
 def install():
     """Routine to be run by the win32 installer with the -install switch."""
-
+    
     from IPython.Release import version
-
+    
     # Get some system constants
     prefix = sys.prefix
-    python = prefix + r'\python.exe'
-    # Lookup path to common startmenu ...
-    ip_dir = get_special_folder_path('CSIDL_COMMON_PROGRAMS') + r'\IPython'
+    python = pjoin(prefix, 'python.exe')
     
-    # Some usability warnings at installation time.  I don't want them at the
-    # top-level, so they don't appear if the user is uninstalling.
-    try:
-        import ctypes
-    except ImportError:
-        print ('To take full advantage of IPython, you need ctypes from:\n'
-               'http://sourceforge.net/projects/ctypes')
-
-    try:
-        import win32con
-    except ImportError:
-        print ('To take full advantage of IPython, you need pywin32 from:\n'
-               'http://starship.python.net/crew/mhammond/win32/Downloads.html')
-
-    try:
-        import readline
-    except ImportError:
-        print ('To take full advantage of IPython, you need readline from:\n'
-               'https://launchpad.net/pyreadline')
-
-    ipybase = '"' + prefix + r'\scripts\ipython"'
+    # Lookup path to common startmenu ...
+    ip_start_menu = pjoin(get_special_folder_path('CSIDL_COMMON_PROGRAMS'), 'IPython')
     # Create IPython entry ...
-    if not os.path.isdir(ip_dir):
-        os.mkdir(ip_dir)
-        directory_created(ip_dir)
-
-    # Create program shortcuts ...
-    f = ip_dir + r'\IPython.lnk'
-    a = ipybase
-    mkshortcut(python,'IPython',f,a)
-
-    f = ip_dir + r'\pysh.lnk'
-    a = ipybase+' -p sh'
-    mkshortcut(python,'IPython (command prompt mode)',f,a)
-
-    f = ip_dir + r'\pylab.lnk'
-    a = ipybase+' -pylab'
-    mkshortcut(python,'IPython (PyLab mode)',f,a)
-
-    f = ip_dir + r'\scipy.lnk'
-    a = ipybase+' -pylab -p scipy'
-    mkshortcut(python,'IPython (scipy profile)',f,a)
-
+    if not os.path.isdir(ip_start_menu):
+        os.mkdir(ip_start_menu)
+        directory_created(ip_start_menu)
+    
+    # Create .py and .bat files to make things available from
+    # the Windows command line.  Thanks to the Twisted project
+    # for this logic!
+    programs = [
+        'ipython',
+        'iptest',
+        'ipcontroller',
+        'ipengine',
+        'ipcluster',
+        'ipythonx',
+        'ipython-wx',
+        'irunner'
+    ]
+    scripts = pjoin(prefix,'scripts')
+    for program in programs:
+        raw = pjoin(scripts, program)
+        bat = raw + '.bat'
+        py = raw + '.py'
+        # Create .py versions of the scripts
+        shutil.copy(raw, py)
+        # Create .bat files for each of the scripts
+        bat_file = file(bat,'w')
+        bat_file.write("@%s %s %%*" % (python, py))
+        bat_file.close()
+            
+    # Now move onto setting the Start Menu up
+    ipybase = pjoin(scripts, 'ipython')
+
+    link = pjoin(ip_start_menu, 'IPython.lnk')
+    cmd = '"%s"' % ipybase
+    mkshortcut(python,'IPython',link,cmd)
+    
+    link = pjoin(ip_start_menu, 'pysh.lnk')
+    cmd = '"%s" -p sh' % ipybase
+    mkshortcut(python,'IPython (command prompt mode)',link,cmd)
+    
+    link = pjoin(ip_start_menu, 'pylab.lnk')
+    cmd = '"%s" -pylab' % ipybase
+    mkshortcut(python,'IPython (PyLab mode)',link,cmd)
+    
+    link = pjoin(ip_start_menu, 'scipy.lnk')
+    cmd = '"%s" -pylab -p scipy' % ipybase
+    mkshortcut(python,'IPython (scipy profile)',link,cmd)
+    
+    link = pjoin(ip_start_menu, 'IPython test suite.lnk')
+    cmd = '"%s" -vv' % pjoin(scripts, 'iptest')
+    mkshortcut(python,'Run the IPython test suite',link,cmd)
+    
+    link = pjoin(ip_start_menu, 'ipcontroller.lnk')
+    cmd = '"%s" -xy' % pjoin(scripts, 'ipcontroller')
+    mkshortcut(python,'IPython controller',link,cmd)
+    
+    link = pjoin(ip_start_menu, 'ipengine.lnk')
+    cmd = '"%s"' % pjoin(scripts, 'ipengine')
+    mkshortcut(python,'IPython engine',link,cmd)
+    
     # Create documentation shortcuts ...
     t = prefix + r'\share\doc\ipython\manual\ipython.pdf'
-    f = ip_dir + r'\Manual in PDF.lnk'
+    f = ip_start_menu + r'\Manual in PDF.lnk'
     mkshortcut(t,r'IPython Manual - PDF-Format',f)
-
+    
     t = prefix + r'\share\doc\ipython\manual\html\index.html'
-    f = ip_dir + r'\Manual in HTML.lnk'
+    f = ip_start_menu + r'\Manual in HTML.lnk'
     mkshortcut(t,'IPython Manual - HTML-Format',f)
-
-    # make ipython.py
-    shutil.copy(prefix + r'\scripts\ipython', prefix + r'\scripts\ipython.py')
+    
     
 def remove():
     """Routine to be run by the win32 installer with the -remove switch."""
     pass
 
 # main()
 if len(sys.argv) > 1:
     if sys.argv[1] == '-install':
         install()
     elif sys.argv[1] == '-remove':
         remove()
     else:
         print "Script was called with option %s" % sys.argv[1]

setupbase.py

@@ -1,278 +1,279 b''
 # encoding: utf-8
 
 """
 This module defines the things that are used in setup.py for building IPython
 
 This includes:
 
     * The basic arguments to setup
     * Functions for finding things like packages, package data, etc.
     * A function for checking dependencies.
 """
 
 __docformat__ = "restructuredtext en"
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
 
 import os, sys
 
 from glob import glob
 
 from setupext import install_data_ext
 
 #-------------------------------------------------------------------------------
 # Useful globals and utility functions
 #-------------------------------------------------------------------------------
 
 # A few handy globals
 isfile = os.path.isfile
 pjoin = os.path.join
 
 def oscmd(s):
     print ">", s
     os.system(s)
 
 # A little utility we'll need below, since glob() does NOT allow you to do
 # exclusion on multiple endings!
 def file_doesnt_endwith(test,endings):
     """Return true if test is a file and its name does NOT end with any
     of the strings listed in endings."""
     if not isfile(test):
         return False
     for e in endings:
         if test.endswith(e):
             return False
     return True
 
 #---------------------------------------------------------------------------
 # Basic project information
 #---------------------------------------------------------------------------
 
 # Release.py contains version, authors, license, url, keywords, etc.
 execfile(pjoin('IPython','Release.py'))
 
 # Create a dict with the basic information
 # This dict is eventually passed to setup after additional keys are added.
 setup_args = dict(
       name             = name,
       version          = version,
       description      = description,
       long_description = long_description,
       author           = author,
       author_email     = author_email,
       url              = url,
       download_url     = download_url,
       license          = license,
       platforms        = platforms,
       keywords         = keywords,
       cmdclass         = {'install_data': install_data_ext},
       )
 
 
 #---------------------------------------------------------------------------
 # Find packages
 #---------------------------------------------------------------------------
 
 def add_package(packages,pname,config=False,tests=False,scripts=False,
                 others=None):
     """
     Add a package to the list of packages, including certain subpackages.
     """
     packages.append('.'.join(['IPython',pname]))
     if config:
         packages.append('.'.join(['IPython',pname,'config']))
     if tests:
         packages.append('.'.join(['IPython',pname,'tests']))
     if scripts:
         packages.append('.'.join(['IPython',pname,'scripts']))
     if others is not None:
         for o in others:
             packages.append('.'.join(['IPython',pname,o]))
 
 def find_packages():
     """
     Find all of IPython's packages.
     """
     packages = ['IPython']
     add_package(packages, 'config', tests=True)
     add_package(packages , 'Extensions')
     add_package(packages, 'external')
     add_package(packages, 'gui')
     add_package(packages, 'gui.wx')
     add_package(packages, 'frontend', tests=True)
     add_package(packages, 'frontend._process')
     add_package(packages, 'frontend.wx')
     add_package(packages, 'frontend.cocoa', tests=True)
     add_package(packages, 'kernel', config=True, tests=True, scripts=True)
     add_package(packages, 'kernel.core', config=True, tests=True)
     add_package(packages, 'testing', tests=True)
+    add_package(packages, 'tests')
     add_package(packages, 'testing.plugin', tests=False)
     add_package(packages, 'tools', tests=True)
     add_package(packages, 'UserConfig')
     return packages
 
 #---------------------------------------------------------------------------
 # Find package data
 #---------------------------------------------------------------------------
 
 def find_package_data():
     """
     Find IPython's package_data.
     """
     # This is not enough for these things to appear in an sdist.
     # We need to muck with the MANIFEST to get this to work
     package_data = {
         'IPython.UserConfig' : ['*'],
         'IPython.tools.tests' : ['*.txt'],
         'IPython.testing' : ['*.txt']
     }
     return package_data
 
 
 #---------------------------------------------------------------------------
 # Find data files
 #---------------------------------------------------------------------------
 
 def make_dir_struct(tag,base,out_base):
     """Make the directory structure of all files below a starting dir.
 
     This is just a convenience routine to help build a nested directory
     hierarchy because distutils is too stupid to do this by itself.
 
     XXX - this needs a proper docstring!
     """
     
     # we'll use these a lot below
     lbase = len(base)
     pathsep = os.path.sep
     lpathsep = len(pathsep)
     
     out = []
     for (dirpath,dirnames,filenames) in os.walk(base):
         # we need to strip out the dirpath from the base to map it to the
         # output (installation) path.  This requires possibly stripping the
         # path separator, because otherwise pjoin will not work correctly
         # (pjoin('foo/','/bar') returns '/bar').
         
         dp_eff = dirpath[lbase:]
         if dp_eff.startswith(pathsep):
             dp_eff = dp_eff[lpathsep:]
         # The output path must be anchored at the out_base marker    
         out_path = pjoin(out_base,dp_eff)
         # Now we can generate the final filenames. Since os.walk only produces
         # filenames, we must join back with the dirpath to get full valid file
         # paths:
         pfiles = [pjoin(dirpath,f) for f in filenames]
         # Finally, generate the entry we need, which is a triple of (tag,output
         # path, files) for use as a data_files parameter in install_data.
         out.append((tag,out_path,pfiles))
 
     return out
     
 
 def find_data_files():
     """
     Find IPython's data_files.
 
     Most of these are docs.
     """
     
     docdirbase  = 'share/doc/ipython'
     manpagebase = 'share/man/man1'
 
     # Simple file lists can be made by hand
     manpages  = filter(isfile, glob('docs/man/*.1.gz'))
     igridhelpfiles = filter(isfile, glob('IPython/Extensions/igrid_help.*'))
 
     # For nested structures, use the utility above
     example_files = make_dir_struct('data','docs/examples',
                                     pjoin(docdirbase,'examples'))
     manual_files = make_dir_struct('data','docs/dist',pjoin(docdirbase,'manual'))
 
     # And assemble the entire output list
     data_files = [ ('data',manpagebase, manpages),
                    ('data',pjoin(docdirbase,'extensions'),igridhelpfiles),
                    ] + manual_files + example_files
                  
     ## import pprint  # dbg
     ## print '*'*80
     ## print 'data files'
     ## pprint.pprint(data_files)
     ## print '*'*80
     
     return data_files
 
 #---------------------------------------------------------------------------
 # Find scripts
 #---------------------------------------------------------------------------
 
 def find_scripts():
     """
     Find IPython's scripts.
     """
     scripts = ['IPython/kernel/scripts/ipengine',
                'IPython/kernel/scripts/ipcontroller',
                'IPython/kernel/scripts/ipcluster',
                'scripts/ipython',
                'scripts/ipythonx',
                'scripts/ipython-wx',
                'scripts/pycolor',
                'scripts/irunner',
                'scripts/iptest',
                ]
     
     # Script to be run by the windows binary installer after the default setup
     # routine, to add shortcuts and similar windows-only things.  Windows
     # post-install scripts MUST reside in the scripts/ dir, otherwise distutils
     # doesn't find them.
     if 'bdist_wininst' in sys.argv:
         if len(sys.argv) > 2 and ('sdist' in sys.argv or 'bdist_rpm' in sys.argv):
             print >> sys.stderr,"ERROR: bdist_wininst must be run alone. Exiting."
             sys.exit(1)
         scripts.append('scripts/ipython_win_post_install.py')
     
     return scripts
 
 #---------------------------------------------------------------------------
 # Verify all dependencies
 #---------------------------------------------------------------------------
 
 def check_for_dependencies():
     """Check for IPython's dependencies.
     
     This function should NOT be called if running under setuptools!
     """
     from setupext.setupext import (
         print_line, print_raw, print_status, print_message,
         check_for_zopeinterface, check_for_twisted,
         check_for_foolscap, check_for_pyopenssl,
         check_for_sphinx, check_for_pygments,
         check_for_nose, check_for_pexpect
     )
     print_line()
     print_raw("BUILDING IPYTHON")
     print_status('python', sys.version)
     print_status('platform', sys.platform)
     if sys.platform == 'win32':
         print_status('Windows version', sys.getwindowsversion())
     
     print_raw("")
     print_raw("OPTIONAL DEPENDENCIES")
 
     check_for_zopeinterface()
     check_for_twisted()
     check_for_foolscap()
     check_for_pyopenssl()
     check_for_sphinx()
     check_for_pygments()
     check_for_nose()
     check_for_pexpect()

setupegg.py

@@ -1,20 +1,14 b''
 #!/usr/bin/env python
 """Wrapper to run setup.py using setuptools."""
 
-import os
 import sys
 
-# Add my local path to sys.path
-home = os.environ['HOME']
-sys.path.insert(0,'%s/usr/local/lib/python%s/site-packages' %
-                (home,sys.version[:3]))
-
 # now, import setuptools and call the actual setup
 import setuptools
 # print sys.argv
 #sys.argv=['','bdist_egg']
 execfile('setup.py')
 
 # clean up the junk left around by setuptools
 if "develop" not in sys.argv:
     os.system('rm -rf ipython.egg-info build')