Unicode fixes (utf-8 used by default if ascii is not enough). This should fix some reported crashes.

Fix broken auto-quoting for pysh.
Remove old 2.1 specific hacks and move all 2.2 compatibility symbols into genutils.
Simplify i/o abstraction layer a bit.
Full details in changelog.

IPython/Extensions/InterpreterExec.py

@@ -5,7 +5,7 b' We define a special input line filter to allow typing lines which begin with'
 '~', '/' or '.'. If one of those strings is encountered, it is automatically
 executed.
 
-$Id: InterpreterExec.py 573 2005-04-08 08:38:09Z fperez $"""
+$Id: InterpreterExec.py 638 2005-07-18 03:01:41Z fperez $"""
 
 #*****************************************************************************
 #       Copyright (C) 2004 W.J. van der Laan <[e-mail]>
@@ -264,7 +264,9 b" __IPYTHON__.magic_unalias('sx')"
 # 'gnome-terminal' are interpreted as a single alias instead of variable
 # 'gnome' minus variable 'terminal'.
 import re
-__IPYTHON__.line_split = re.compile(r'^(\s*)([\?\w\.\-\+]+\w*\s*)(\(?.*$)')
+__IPYTHON__.line_split = re.compile(r'^([\s*,;/])'
+                                    r'([\?\w\.\-\+]+\w*\s*)'
+                                    r'(\(?.*$)')
 
 # Namespace cleanup
 del re

IPython/FlexCompleter.py

@@ -84,13 +84,6 b' import __main__'
 
 __all__ = ["Completer"]
 
-# declares Python 2.2 compatibility symbols:
-try:
-    basestring
-except NameError:
-    import types
-    basestring = (types.StringType, types.UnicodeType)
-
 class Completer:
     def __init__(self, namespace = None):
         """Create a new completer for the command line.

IPython/Itpl.py

@@ -44,7 +44,7 b' each time the instance is evaluated with str(instance). For example:'
     foo = "bar"
     print str(s)
 
-$Id: Itpl.py 542 2005-03-18 09:16:04Z fperez $
+$Id: Itpl.py 638 2005-07-18 03:01:41Z fperez $
 """                   # ' -> close an open quote for stupid emacs
 
 #*****************************************************************************
@@ -103,8 +103,9 b' class Itpl:'
     evaluation and substitution happens in the namespace of the
     caller when str(instance) is called."""
 
-    def __init__(self, format):
-        """The single argument to this constructor is a format string.
+    def __init__(self, format,codec='utf_8',encoding_errors='backslashreplace'):
+        """The single mandatory argument to this constructor is a format
+        string.
 
         The format string is parsed according to the following rules:
 
@@ -119,12 +120,25 b' class Itpl:'
             a Python expression.
 
         3.  Outside of the expressions described in the above two rules,
-            two dollar signs in a row give you one literal dollar sign."""
+            two dollar signs in a row give you one literal dollar sign.
 
-        if type(format) != StringType:
+        Optional arguments:
+
+        - codec('utf_8'): a string containing the name of a valid Python
+        codec.
+
+        - encoding_errors('backslashreplace'): a string with a valid error handling
+        policy.  See the codecs module documentation for details.
+
+        These are used to encode the format string if a call to str() fails on
+        the expanded result."""
+
+        if not isinstance(format,basestring):
             raise TypeError, "needs string initializer"
         self.format = format
-
+        self.codec = codec
+        self.encoding_errors = encoding_errors
+        
         namechars = "abcdefghijklmnopqrstuvwxyz" \
             "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_";
         chunks = []
@@ -174,6 +188,23 b' class Itpl:'
     def __repr__(self):
         return "<Itpl %s >" % repr(self.format)
 
+    def _str(self,glob,loc):
+        """Evaluate to a string in the given globals/locals.
+
+        The final output is built by calling str(), but if this fails, the
+        result is encoded with the instance's codec and error handling policy,
+        via a call to out.encode(self.codec,self.encoding_errors)"""
+        result = []
+        app = result.append
+        for live, chunk in self.chunks:
+            if live: app(str(eval(chunk,glob,loc)))
+            else: app(chunk)
+        out = ''.join(result)
+        try:
+            return str(out)
+        except UnicodeError:
+            return out.encode(self.codec,self.encoding_errors)
+
     def __str__(self):
         """Evaluate and substitute the appropriate parts of the string."""
 
@@ -183,13 +214,8 b' class Itpl:'
         while frame.f_globals["__name__"] == __name__: frame = frame.f_back
         loc, glob = frame.f_locals, frame.f_globals
 
-        result = []
-        for live, chunk in self.chunks:
-            if live: result.append(str(eval(chunk,glob,loc)))
-            else: result.append(chunk)
-
-        return ''.join(result)
-
+        return self._str(glob,loc)
+    
 class ItplNS(Itpl):
     """Class representing a string with interpolation abilities.
 
@@ -199,7 +225,8 b' class ItplNS(Itpl):'
     caller to supply a different namespace for the interpolation to occur than
     its own."""
     
-    def __init__(self, format,globals,locals=None):
+    def __init__(self, format,globals,locals=None,
+                 codec='utf_8',encoding_errors='backslashreplace'):
         """ItplNS(format,globals[,locals]) -> interpolating string instance.
 
         This constructor, besides a format string, takes a globals dictionary
@@ -211,17 +238,14 b' class ItplNS(Itpl):'
             locals = globals
         self.globals = globals
         self.locals = locals
-        Itpl.__init__(self,format)
+        Itpl.__init__(self,format,codec,encoding_errors)
         
     def __str__(self):
         """Evaluate and substitute the appropriate parts of the string."""
-        glob = self.globals
-        loc = self.locals
-        result = []
-        for live, chunk in self.chunks:
-            if live: result.append(str(eval(chunk,glob,loc)))
-            else: result.append(chunk)
-        return ''.join(result)
+        return self._str(self.globals,self.locals)
+
+    def __repr__(self):
+        return "<ItplNS %s >" % repr(self.format)
 
 # utilities for fast printing
 def itpl(text): return str(Itpl(text))

IPython/Prompts.py

@@ -2,7 +2,7 b''
 """
 Classes for handling input/output prompts.
 
-$Id: Prompts.py 564 2005-03-26 22:43:58Z fperez $"""
+$Id: Prompts.py 638 2005-07-18 03:01:41Z fperez $"""
 
 #*****************************************************************************
 #       Copyright (C) 2001-2004 Fernando Perez <[e-mail]>
@@ -194,11 +194,19 b' def str_safe(arg):'
 
     If str(arg) fails, <ERROR: ... > is returned, where ... is the exception
     error message."""
-    
+
     try:
-        return str(arg)
+        out = str(arg)
+    except UnicodeError:
+        try:
+            out = arg.encode('utf_8','replace')
+        except Exception,msg:
+            # let's keep this little duplication here, so that the most common
+            # case doesn't suffer from a double try wrapping.
+            out = '<ERROR: %s>' % msg
     except Exception,msg:
-        return '<ERROR: %s>' % msg
+        out = '<ERROR: %s>' % msg
+    return out
 
 class BasePrompt:
     """Interactive prompt similar to Mathematica's."""
@@ -413,12 +421,12 b' class CachedOutput:'
         self.ps2_str = self._set_prompt_str(ps2,'   .\\D.: ','... ')
         self.ps_out_str = self._set_prompt_str(ps_out,'Out[\\#]: ','')
 
+        self.color_table = PromptColors
         self.prompt1 = Prompt1(self,sep=input_sep,prompt=self.ps1_str,
                                pad_left=pad_left)
         self.prompt2 = Prompt2(self,prompt=self.ps2_str,pad_left=pad_left)
         self.prompt_out = PromptOut(self,sep='',prompt=self.ps_out_str,
                                     pad_left=pad_left)
-        self.color_table = PromptColors
         self.set_colors(colors)
 
         # other more normal stuff
@@ -480,15 +488,16 b' class CachedOutput:'
             except KeyError:
                 pass
         if arg is not None:
+            cout_write = Term.cout.write # fast lookup
             # first handle the cache and counters
             self.update(arg)
             # do not print output if input ends in ';'
             if self.input_hist[self.prompt_count].endswith(';\n'):
                 return
             # don't use print, puts an extra space
-            Term.cout.write(self.output_sep)
+            cout_write(self.output_sep)
             if self.do_full_cache:
-                Term.cout.write(str(self.prompt_out))
+                cout_write(str(self.prompt_out))
 
             if isinstance(arg,Macro):
                 print 'Executing Macro...'
@@ -499,7 +508,7 b' class CachedOutput:'
 
             # and now call a possibly user-defined print mechanism
             self.display(arg)
-            Term.cout.write(self.output_sep2)
+            cout_write(self.output_sep2)
             Term.cout.flush()
 
     def _display(self,arg):
@@ -509,18 +518,7 b' class CachedOutput:'
         of certain types of output."""
 
         if self.Pprint:
-            # The following is an UGLY kludge, b/c python fails to properly
-            # identify instances of classes imported in the user namespace
-            # (they have different memory locations, I guess). Structs are
-            # essentially dicts but pprint doesn't know what to do with them.
-            try:
-                if arg.__class__.__module__ == 'Struct' and \
-                   arg.__class__.__name__ == 'Struct':
-                    out = 'Struct:\n%s' % pformat(arg.dict())
-                else:
-                    out = pformat(arg)
-            except:
-                out = pformat(arg)
+            out = pformat(arg)
             if '\n' in out:
                 # So that multi-line strings line up with the left column of
                 # the screen, instead of having the output prompt mess up

IPython/Struct.py

@@ -1,7 +1,7 b''
 # -*- coding: utf-8 -*-
 """Mimic C structs with lots of extra functionality.
 
-$Id: Struct.py 410 2004-11-04 07:58:17Z fperez $"""
+$Id: Struct.py 638 2005-07-18 03:01:41Z fperez $"""
 
 #*****************************************************************************
 #       Copyright (C) 2001-2004 Fernando Perez <[e-mail]>
@@ -315,9 +315,8 b' class Struct:'
                     del inv_conflict_solve_user[name]
             conflict_solve.update(Struct.__dict_invert(self,inv_conflict_solve_user))
         #print 'merge. conflict_solve: '; pprint(conflict_solve) # dbg
-        # after Python 2.2, use iterators: for key in data_dict will then work
         #print '*'*50,'in merger. conflict_solver:';  pprint(conflict_solve)
-        for key in data_dict.keys():
+        for key in data_dict:
             if key not in self:
                 self[key] = data_dict[key]
             else:

IPython/background_jobs.py

@@ -18,7 +18,7 b' http://folk.uio.no/hpl/scripting'
 (although ultimately no code from this text was used, as IPython's system is a
 separate implementation).
 
-$Id: background_jobs.py 515 2005-02-15 07:41:41Z fperez $
+$Id: background_jobs.py 638 2005-07-18 03:01:41Z fperez $
 """
 
 #*****************************************************************************
@@ -38,15 +38,6 b' import threading,sys'
 from IPython.ultraTB import AutoFormattedTB
 from IPython.genutils import warn,error
 
-# declares Python 2.2 compatibility symbols:
-try:
-    basestring
-except NameError:
-    import types
-    basestring = (types.StringType, types.UnicodeType)
-    True = 1==1
-    False = 1==0
-
 class BackgroundJobManager:
     """Class to manage a pool of backgrounded threaded jobs.
 

IPython/genutils.py

@@ -5,7 +5,7 b' General purpose utilities.'
 This is a grab-bag of stuff I find useful in most programs I write. Some of
 these things are also convenient when working at the command line.
 
-$Id: genutils.py 633 2005-07-17 01:03:15Z tzanko $"""
+$Id: genutils.py 638 2005-07-18 03:01:41Z fperez $"""
 
 #*****************************************************************************
 #       Copyright (C) 2001-2004 Fernando Perez. <[e-mail]>
@@ -23,9 +23,33 b' __license__ = Release.license'
 import __main__
 import types,commands,time,sys,os,re,shutil
 import tempfile
+import codecs
 from IPython.Itpl import Itpl,itpl,printpl
 from IPython import DPyGetOpt
 
+# Build objects which appeared in Python 2.3 for 2.2, to make ipython
+# 2.2-friendly
+try:
+    basestring
+except NameError:
+    import types
+    basestring = (types.StringType, types.UnicodeType)
+    True = 1==1
+    False = 1==0
+
+    def enumerate(obj):
+        i = -1
+        for item in obj:
+            i += 1
+            yield i, item
+
+    # add these to the builtin namespace, so that all modules find them
+    import __builtin__
+    __builtin__.basestring = basestring
+    __builtin__.True = True
+    __builtin__.False = False
+    __builtin__.enumerate = enumerate
+
 #****************************************************************************
 # Exceptions
 class Error(Exception):
@@ -33,26 +57,29 b' class Error(Exception):'
     pass
 
 #----------------------------------------------------------------------------
-class Stream:
-    """Simple class to hold the various I/O streams in Term"""
-
-    def __init__(self,stream,name):
+class IOStream:
+    def __init__(self,stream,fallback):
+        if not hasattr(stream,'write') or not hasattr(stream,'flush'):
+            stream = fallback
         self.stream = stream
-        self.name = name
-        try:
-            self.fileno = stream.fileno()
-        except AttributeError:
-            msg = ("Stream <%s> looks suspicious: it lacks a 'fileno' attribute."
-                   % name)
-            print >> sys.stderr, 'WARNING:',msg
-        try:
-            self.mode = stream.mode
-        except AttributeError:
-            msg = ("Stream <%s> looks suspicious: it lacks a 'mode' attribute."
-                   % name)
-            print >> sys.stderr, 'WARNING:',msg
+        self._swrite = stream.write
+        self.flush = stream.flush
 
-class Term:
+    def write(self,data):
+        try:
+            self._swrite(data)
+        except:
+            try:
+                # print handles some unicode issues which may trip a plain
+                # write() call.  Attempt to emulate write() by using a
+                # trailing comma
+                print >> self.stream, data,
+            except:
+                # if we get here, something is seriously broken.
+                print >> sys.stderr, \
+                      'ERROR - failed to write data to stream:', stream
+        
+class IOTerm:
     """ Term holds the file or file-like objects for handling I/O operations.
 
     These are normally just sys.stdin, sys.stdout and sys.stderr but for
@@ -62,51 +89,13 b' class Term:'
     # In the future, having IPython channel all its I/O operations through
     # this class will make it easier to embed it into other environments which
     # are not a normal terminal (such as a GUI-based shell)
-    in_s  = Stream(sys.stdin,'cin')
-    out_s = Stream(sys.stdout,'cout')
-    err_s = Stream(sys.stderr,'cerr')
-
-    # Store the three streams in (err,out,in) order so that if we need to reopen
-    # them, the error channel is reopened first to provide info.
-    streams = [err_s,out_s,in_s]
-
-    # The class globals should be the actual 'bare' streams for normal I/O to work
-    cin  = streams[2].stream
-    cout = streams[1].stream
-    cerr = streams[0].stream
-    
-    def reopen_all(cls):
-        """Reopen all streams if necessary.
-
-        This should only be called if it is suspected that someting closed
-        accidentally one of the I/O streams."""
-
-        any_closed = 0
-
-        for sn in range(len(cls.streams)):
-            st = cls.streams[sn]
-            if st.stream.closed:
-                any_closed = 1
-                new_stream = os.fdopen(os.dup(st.fileno), st.mode,0)
-                cls.streams[sn] = Stream(new_stream,st.name)
-                print >> cls.streams[0].stream, \
-                      '\nWARNING:\nStream Term.%s had to be reopened!' % st.name
-
-        # Rebuild the class globals
-        cls.cin = cls.streams[2].stream
-        cls.cout = cls.streams[1].stream
-        cls.cerr = cls.streams[0].stream
-
-    reopen_all = classmethod(reopen_all)
-
-    def set_stdout(cls,stream):
-        """Set the stream """
-        cls.cout = stream
-    set_stdout = classmethod(set_stdout)
-
-    def set_stderr(cls,stream):
-        cls.cerr = stream
-    set_stderr = classmethod(set_stderr)
+    def __init__(self,cin=None,cout=None,cerr=None):
+        self.cin  = IOStream(cin,sys.stdin)
+        self.cout = IOStream(cout,sys.stdout)
+        self.cerr = IOStream(cerr,sys.stderr)
+        
+# Global variable to be used for all I/O
+Term = IOTerm()
 
 # Windows-specific code to load Gary Bishop's readline and configure it
 # automatically for the users
@@ -123,8 +112,8 b" if os.name == 'nt':"
         except AttributeError:
             pass
         else:
-            Term.set_stdout(_out)
-            Term.set_stderr(_out)
+            # Remake Term to use the readline i/o facilities
+            Term = IOTerm(cout=_out,cerr=_out)
             del _out
 
 #****************************************************************************

IPython/iplib.py

@@ -6,7 +6,7 b' Requires Python 2.1 or newer.'
 
 This file contains all the classes and helper functions specific to IPython.
 
-$Id: iplib.py 602 2005-06-09 03:02:30Z fperez $
+$Id: iplib.py 638 2005-07-18 03:01:41Z fperez $
 """
 
 #*****************************************************************************
@@ -69,15 +69,6 b' from IPython.genutils import *'
 # overwrites it (like wx.py.PyShell does)
 raw_input_original = raw_input
 
-# declares Python 2.2 compatibility symbols:
-try:
-    enumerate
-except NameError:
-    def enumerate(obj):
-        i = -1
-        for item in obj:
-            i += 1
-            yield i, item
 #****************************************************************************
 # Some utility function definitions
 
@@ -1337,7 +1328,7 b' want to merge them back into the new files.""" % locals()'
         self.interact(header)
 
         # Remove locals from namespace
-        for k in local_ns.keys():
+        for k in local_ns:
             del self.user_ns[k]
 
     def interact(self, banner=None):
@@ -1363,10 +1354,9 b' want to merge them back into the new files.""" % locals()'
 
         # Mark activity in the builtins
         __builtin__.__dict__['__IPYTHON__active'] += 1
-        while 1:
-            # This is set by a call to %Exit or %Quit
-            if self.exit_now:
-                break
+
+        # exit_now is set by a call to %Exit or %Quit
+        while not self.exit_now:
             try:
                 if more:
                     prompt = self.outputcache.prompt2
@@ -1421,82 +1411,7 b' want to merge them back into the new files.""" % locals()'
                      "Because of how pdb handles the stack, it is impossible\n"
                      "for IPython to properly format this particular exception.\n"
                      "IPython will resume normal operation.")
-            except:
-                # We should never get here except in fairly bizarre situations
-                # (or b/c of an IPython bug). One reasonable exception is if
-                # the user sets stdin/out/err to a broken object (or closes
-                # any of them!)
-
-                fixed_in_out_err = 0
-
-                # Call the Term I/O class and have it reopen any stream which
-                # the user might have closed.
-                Term.reopen_all()
-
-                # Do the same manually for sys.stderr/out/in
-                
-                # err first, so we can print at least warnings
-                if sys.__stderr__.closed:
-                    sys.__stderr__ = os.fdopen(os.dup(2),'w',0)
-                    fixed_err_err = 1
-                    print >> sys.__stderr__,"""
-WARNING:
-sys.__stderr__ was closed!
-I've tried to reopen it, but bear in mind that things may not work normally
-from now.  In particular, readline support may have broken.
-"""
-                # Next, check stdin/out    
-                if sys.__stdin__.closed:
-                    sys.__stdin__ = os.fdopen(os.dup(0),'r',0)
-                    fixed_in_out_err = 1
-                    print >> sys.__stderr__,"""
-WARNING:
-sys.__stdin__ was closed!
-I've tried to reopen it, but bear in mind that things may not work normally
-from now.  In particular, readline support may have broken.
-"""
-                if sys.__stdout__.closed:
-                    sys.__stdout__ = os.fdopen(os.dup(1),'w',0)
-                    fixed_in_out_err = 1
-                    print >> sys.__stderr__,"""
-WARNING:
-sys.__stdout__ was closed!
-I've tried to reopen it, but bear in mind that things may not work normally
-from now.  In particular, readline support may have broken.
-"""
-
-                # Now, check mismatch of objects
-                if sys.stdin is not sys.__stdin__:
-                    sys.stdin = sys.__stdin__
-                    fixed_in_out_err = 1
-                    print >> sys.__stderr__,"""
-WARNING:
-sys.stdin has been reset to sys.__stdin__.
-There seemed to be a problem with your sys.stdin.
-"""
-                if sys.stdout is not sys.__stdout__:
-                    sys.stdout = sys.__stdout__
-                    fixed_in_out_err = 1
-                    print >> sys.__stderr__,"""
-WARNING:
-sys.stdout has been reset to sys.__stdout__.
-There seemed to be a problem with your sys.stdout.
-"""
-
-                if sys.stderr is not sys.__stderr__:
-                    sys.stderr = sys.__stderr__
-                    fixed_in_out_err = 1
-                    print >> sys.__stderr__,"""
-WARNING:
-sys.stderr has been reset to sys.__stderr__.
-There seemed to be a problem with your sys.stderr.
-"""
-                # If the problem wasn't a broken out/err, it's an IPython bug
-                # I wish we could ask the user whether to crash or not, but
-                # calling any function at this point messes up the stack.
-                if not fixed_in_out_err:
-                    raise
-                
+            
         # We are off again...
         __builtin__.__dict__['__IPYTHON__active'] -= 1
 

IPython/ipmaker.py

@@ -6,7 +6,7 b' Requires Python 2.1 or better.'
 
 This file contains the main make_IPython() starter function.
 
-$Id: ipmaker.py 582 2005-05-13 21:20:00Z fperez $"""
+$Id: ipmaker.py 638 2005-07-18 03:01:41Z fperez $"""
 
 #*****************************************************************************
 #       Copyright (C) 2001-2004 Fernando Perez. <[e-mail]>
@@ -85,19 +85,8 b' def make_IPython(argv=None,user_ns=None,debug=1,rc_override=None,'
     IP = shell_class('__IP',user_ns=user_ns,**kw)
 
     # Put 'help' in the user namespace
-    try:
-        from site import _Helper
-    except ImportError:
-        # Use the _Helper class from Python 2.2 for older Python versions
-        class _Helper:
-            def __repr__(self):
-                return "Type help() for interactive help, " \
-                       "or help(object) for help about object."
-            def __call__(self, *args, **kwds):
-                import pydoc
-                return pydoc.help(*args, **kwds)
-    else:
-        IP.user_ns['help'] = _Helper()
+    from site import _Helper
+    IP.user_ns['help'] = _Helper()
 
     if DEVDEBUG:
         # For developer debugging only (global flag)

doc/ChangeLog

@@ -1,3 +1,39 b''
[phone]  Fernando Perez  <[e-mail]>
+
+	* IPython/Prompts.py (str_safe): Make unicode-safe.  Also remove
+	some old hacks and clean up a bit other routines; code should be
+	simpler and a bit faster.
+
+	* IPython/iplib.py (interact): removed some last-resort attempts
+	to survive broken stdout/stderr.  That code was only making it
+	harder to abstract out the i/o (necessary for gui integration),
+	and the crashes it could prevent were extremely rare in practice
+	(besides being fully user-induced in a pretty violent manner).
+
+	* IPython/genutils.py (IOStream.__init__): Simplify the i/o stuff.
+	Nothing major yet, but the code is simpler to read; this should
+	make it easier to do more serious modifications in the future.
+
+	* IPython/Extensions/InterpreterExec.py: Fix auto-quoting in pysh,
+	which broke in .15 (thanks to a report by Ville).
+
+	* IPython/Itpl.py (Itpl.__init__): add unicode support (it may not
+	be quite correct, I know next to nothing about unicode).  This
+	will allow unicode strings to be used in prompts, amongst other
+	cases.  It also will prevent ipython from crashing when unicode
+	shows up unexpectedly in many places.  If ascii encoding fails, we
+	assume utf_8.  Currently the encoding is not a user-visible
+	setting, though it could be made so if there is demand for it.
+
+	* IPython/ipmaker.py (make_IPython): remove old 2.1-specific hack.
+
+	* IPython/Struct.py (Struct.merge): switch keys() to iterator.
+
+	* IPython/background_jobs.py: moved 2.2 compatibility to genutils.
+
+	* IPython/genutils.py: Add 2.2 compatibility here, so all other
+	code can work transparently for 2.2/2.3.
+
 2005-07-16  Fernando Perez  <[e-mail]>
 
 	* IPython/ultraTB.py (ExceptionColors): Make a global variable
@@ -11,7 +47,7 b''
 	slightly modified version of the patch in
 	http://www.scipy.net/roundup/ipython/issue34, which also allows me
 	to remove the previous try/except solution (which was costlier).
-	Thanks to glehmann for the fix.
+	Thanks to Gaetan Lehmann <gaetan.lehmann AT jouy.inra.fr> for the fix.
 
 2005-06-08  Fernando Perez  <[e-mail]>