From c83aafa375965e8a1a3115cdcd2e97ca5405e677 2005-07-18 03:01:41 From: fperez Date: 2005-07-18 03:01:41 Subject: [PATCH] Unicode fixes (utf-8 used by default if ascii is not enough). This should fix some reported crashes. Fix broken auto-quoting for pysh. Remove old 2.1 specific hacks and move all 2.2 compatibility symbols into genutils. Simplify i/o abstraction layer a bit. Full details in changelog. --- diff --git a/IPython/Extensions/InterpreterExec.py b/IPython/Extensions/InterpreterExec.py index f4fb63a..4204b7d 100644 --- a/IPython/Extensions/InterpreterExec.py +++ b/IPython/Extensions/InterpreterExec.py @@ -5,7 +5,7 @@ We define a special input line filter to allow typing lines which begin with '~', '/' or '.'. If one of those strings is encountered, it is automatically executed. -$Id: InterpreterExec.py 573 2005-04-08 08:38:09Z fperez $""" +$Id: InterpreterExec.py 638 2005-07-18 03:01:41Z fperez $""" #***************************************************************************** # Copyright (C) 2004 W.J. van der Laan @@ -264,7 +264,9 @@ __IPYTHON__.magic_unalias('sx') # 'gnome-terminal' are interpreted as a single alias instead of variable # 'gnome' minus variable 'terminal'. import re -__IPYTHON__.line_split = re.compile(r'^(\s*)([\?\w\.\-\+]+\w*\s*)(\(?.*$)') +__IPYTHON__.line_split = re.compile(r'^([\s*,;/])' + r'([\?\w\.\-\+]+\w*\s*)' + r'(\(?.*$)') # Namespace cleanup del re diff --git a/IPython/FlexCompleter.py b/IPython/FlexCompleter.py index 477d35c..0a0f777 100644 --- a/IPython/FlexCompleter.py +++ b/IPython/FlexCompleter.py @@ -84,13 +84,6 @@ import __main__ __all__ = ["Completer"] -# declares Python 2.2 compatibility symbols: -try: - basestring -except NameError: - import types - basestring = (types.StringType, types.UnicodeType) - class Completer: def __init__(self, namespace = None): """Create a new completer for the command line. diff --git a/IPython/Itpl.py b/IPython/Itpl.py index c62ab4f..a551dac 100644 --- a/IPython/Itpl.py +++ b/IPython/Itpl.py @@ -44,7 +44,7 @@ each time the instance is evaluated with str(instance). For example: foo = "bar" print str(s) -$Id: Itpl.py 542 2005-03-18 09:16:04Z fperez $ +$Id: Itpl.py 638 2005-07-18 03:01:41Z fperez $ """ # ' -> close an open quote for stupid emacs #***************************************************************************** @@ -103,8 +103,9 @@ class Itpl: evaluation and substitution happens in the namespace of the caller when str(instance) is called.""" - def __init__(self, format): - """The single argument to this constructor is a format string. + def __init__(self, format,codec='utf_8',encoding_errors='backslashreplace'): + """The single mandatory argument to this constructor is a format + string. The format string is parsed according to the following rules: @@ -119,12 +120,25 @@ class Itpl: a Python expression. 3. Outside of the expressions described in the above two rules, - two dollar signs in a row give you one literal dollar sign.""" + two dollar signs in a row give you one literal dollar sign. - if type(format) != StringType: + Optional arguments: + + - codec('utf_8'): a string containing the name of a valid Python + codec. + + - encoding_errors('backslashreplace'): a string with a valid error handling + policy. See the codecs module documentation for details. + + These are used to encode the format string if a call to str() fails on + the expanded result.""" + + if not isinstance(format,basestring): raise TypeError, "needs string initializer" self.format = format - + self.codec = codec + self.encoding_errors = encoding_errors + namechars = "abcdefghijklmnopqrstuvwxyz" \ "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_"; chunks = [] @@ -174,6 +188,23 @@ class Itpl: def __repr__(self): return "" % repr(self.format) + def _str(self,glob,loc): + """Evaluate to a string in the given globals/locals. + + The final output is built by calling str(), but if this fails, the + result is encoded with the instance's codec and error handling policy, + via a call to out.encode(self.codec,self.encoding_errors)""" + result = [] + app = result.append + for live, chunk in self.chunks: + if live: app(str(eval(chunk,glob,loc))) + else: app(chunk) + out = ''.join(result) + try: + return str(out) + except UnicodeError: + return out.encode(self.codec,self.encoding_errors) + def __str__(self): """Evaluate and substitute the appropriate parts of the string.""" @@ -183,13 +214,8 @@ class Itpl: while frame.f_globals["__name__"] == __name__: frame = frame.f_back loc, glob = frame.f_locals, frame.f_globals - result = [] - for live, chunk in self.chunks: - if live: result.append(str(eval(chunk,glob,loc))) - else: result.append(chunk) - - return ''.join(result) - + return self._str(glob,loc) + class ItplNS(Itpl): """Class representing a string with interpolation abilities. @@ -199,7 +225,8 @@ class ItplNS(Itpl): caller to supply a different namespace for the interpolation to occur than its own.""" - def __init__(self, format,globals,locals=None): + def __init__(self, format,globals,locals=None, + codec='utf_8',encoding_errors='backslashreplace'): """ItplNS(format,globals[,locals]) -> interpolating string instance. This constructor, besides a format string, takes a globals dictionary @@ -211,17 +238,14 @@ class ItplNS(Itpl): locals = globals self.globals = globals self.locals = locals - Itpl.__init__(self,format) + Itpl.__init__(self,format,codec,encoding_errors) def __str__(self): """Evaluate and substitute the appropriate parts of the string.""" - glob = self.globals - loc = self.locals - result = [] - for live, chunk in self.chunks: - if live: result.append(str(eval(chunk,glob,loc))) - else: result.append(chunk) - return ''.join(result) + return self._str(self.globals,self.locals) + + def __repr__(self): + return "" % repr(self.format) # utilities for fast printing def itpl(text): return str(Itpl(text)) diff --git a/IPython/Prompts.py b/IPython/Prompts.py index 361e8ac..b66441c 100644 --- a/IPython/Prompts.py +++ b/IPython/Prompts.py @@ -2,7 +2,7 @@ """ Classes for handling input/output prompts. -$Id: Prompts.py 564 2005-03-26 22:43:58Z fperez $""" +$Id: Prompts.py 638 2005-07-18 03:01:41Z fperez $""" #***************************************************************************** # Copyright (C) 2001-2004 Fernando Perez @@ -194,11 +194,19 @@ def str_safe(arg): If str(arg) fails, is returned, where ... is the exception error message.""" - + try: - return str(arg) + out = str(arg) + except UnicodeError: + try: + out = arg.encode('utf_8','replace') + except Exception,msg: + # let's keep this little duplication here, so that the most common + # case doesn't suffer from a double try wrapping. + out = '' % msg except Exception,msg: - return '' % msg + out = '' % msg + return out class BasePrompt: """Interactive prompt similar to Mathematica's.""" @@ -413,12 +421,12 @@ class CachedOutput: self.ps2_str = self._set_prompt_str(ps2,' .\\D.: ','... ') self.ps_out_str = self._set_prompt_str(ps_out,'Out[\\#]: ','') + self.color_table = PromptColors self.prompt1 = Prompt1(self,sep=input_sep,prompt=self.ps1_str, pad_left=pad_left) self.prompt2 = Prompt2(self,prompt=self.ps2_str,pad_left=pad_left) self.prompt_out = PromptOut(self,sep='',prompt=self.ps_out_str, pad_left=pad_left) - self.color_table = PromptColors self.set_colors(colors) # other more normal stuff @@ -480,15 +488,16 @@ class CachedOutput: except KeyError: pass if arg is not None: + cout_write = Term.cout.write # fast lookup # first handle the cache and counters self.update(arg) # do not print output if input ends in ';' if self.input_hist[self.prompt_count].endswith(';\n'): return # don't use print, puts an extra space - Term.cout.write(self.output_sep) + cout_write(self.output_sep) if self.do_full_cache: - Term.cout.write(str(self.prompt_out)) + cout_write(str(self.prompt_out)) if isinstance(arg,Macro): print 'Executing Macro...' @@ -499,7 +508,7 @@ class CachedOutput: # and now call a possibly user-defined print mechanism self.display(arg) - Term.cout.write(self.output_sep2) + cout_write(self.output_sep2) Term.cout.flush() def _display(self,arg): @@ -509,18 +518,7 @@ class CachedOutput: of certain types of output.""" if self.Pprint: - # The following is an UGLY kludge, b/c python fails to properly - # identify instances of classes imported in the user namespace - # (they have different memory locations, I guess). Structs are - # essentially dicts but pprint doesn't know what to do with them. - try: - if arg.__class__.__module__ == 'Struct' and \ - arg.__class__.__name__ == 'Struct': - out = 'Struct:\n%s' % pformat(arg.dict()) - else: - out = pformat(arg) - except: - out = pformat(arg) + out = pformat(arg) if '\n' in out: # So that multi-line strings line up with the left column of # the screen, instead of having the output prompt mess up diff --git a/IPython/Struct.py b/IPython/Struct.py index da316f9..4c4f125 100644 --- a/IPython/Struct.py +++ b/IPython/Struct.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- """Mimic C structs with lots of extra functionality. -$Id: Struct.py 410 2004-11-04 07:58:17Z fperez $""" +$Id: Struct.py 638 2005-07-18 03:01:41Z fperez $""" #***************************************************************************** # Copyright (C) 2001-2004 Fernando Perez @@ -315,9 +315,8 @@ class Struct: del inv_conflict_solve_user[name] conflict_solve.update(Struct.__dict_invert(self,inv_conflict_solve_user)) #print 'merge. conflict_solve: '; pprint(conflict_solve) # dbg - # after Python 2.2, use iterators: for key in data_dict will then work #print '*'*50,'in merger. conflict_solver:'; pprint(conflict_solve) - for key in data_dict.keys(): + for key in data_dict: if key not in self: self[key] = data_dict[key] else: diff --git a/IPython/background_jobs.py b/IPython/background_jobs.py index aaa2c81..e428b47 100644 --- a/IPython/background_jobs.py +++ b/IPython/background_jobs.py @@ -18,7 +18,7 @@ http://folk.uio.no/hpl/scripting (although ultimately no code from this text was used, as IPython's system is a separate implementation). -$Id: background_jobs.py 515 2005-02-15 07:41:41Z fperez $ +$Id: background_jobs.py 638 2005-07-18 03:01:41Z fperez $ """ #***************************************************************************** @@ -38,15 +38,6 @@ import threading,sys from IPython.ultraTB import AutoFormattedTB from IPython.genutils import warn,error -# declares Python 2.2 compatibility symbols: -try: - basestring -except NameError: - import types - basestring = (types.StringType, types.UnicodeType) - True = 1==1 - False = 1==0 - class BackgroundJobManager: """Class to manage a pool of backgrounded threaded jobs. diff --git a/IPython/genutils.py b/IPython/genutils.py index ce064b2..4872d26 100644 --- a/IPython/genutils.py +++ b/IPython/genutils.py @@ -5,7 +5,7 @@ General purpose utilities. This is a grab-bag of stuff I find useful in most programs I write. Some of these things are also convenient when working at the command line. -$Id: genutils.py 633 2005-07-17 01:03:15Z tzanko $""" +$Id: genutils.py 638 2005-07-18 03:01:41Z fperez $""" #***************************************************************************** # Copyright (C) 2001-2004 Fernando Perez. @@ -23,9 +23,33 @@ __license__ = Release.license import __main__ import types,commands,time,sys,os,re,shutil import tempfile +import codecs from IPython.Itpl import Itpl,itpl,printpl from IPython import DPyGetOpt +# Build objects which appeared in Python 2.3 for 2.2, to make ipython +# 2.2-friendly +try: + basestring +except NameError: + import types + basestring = (types.StringType, types.UnicodeType) + True = 1==1 + False = 1==0 + + def enumerate(obj): + i = -1 + for item in obj: + i += 1 + yield i, item + + # add these to the builtin namespace, so that all modules find them + import __builtin__ + __builtin__.basestring = basestring + __builtin__.True = True + __builtin__.False = False + __builtin__.enumerate = enumerate + #**************************************************************************** # Exceptions class Error(Exception): @@ -33,26 +57,29 @@ class Error(Exception): pass #---------------------------------------------------------------------------- -class Stream: - """Simple class to hold the various I/O streams in Term""" - - def __init__(self,stream,name): +class IOStream: + def __init__(self,stream,fallback): + if not hasattr(stream,'write') or not hasattr(stream,'flush'): + stream = fallback self.stream = stream - self.name = name - try: - self.fileno = stream.fileno() - except AttributeError: - msg = ("Stream <%s> looks suspicious: it lacks a 'fileno' attribute." - % name) - print >> sys.stderr, 'WARNING:',msg - try: - self.mode = stream.mode - except AttributeError: - msg = ("Stream <%s> looks suspicious: it lacks a 'mode' attribute." - % name) - print >> sys.stderr, 'WARNING:',msg + self._swrite = stream.write + self.flush = stream.flush -class Term: + def write(self,data): + try: + self._swrite(data) + except: + try: + # print handles some unicode issues which may trip a plain + # write() call. Attempt to emulate write() by using a + # trailing comma + print >> self.stream, data, + except: + # if we get here, something is seriously broken. + print >> sys.stderr, \ + 'ERROR - failed to write data to stream:', stream + +class IOTerm: """ Term holds the file or file-like objects for handling I/O operations. These are normally just sys.stdin, sys.stdout and sys.stderr but for @@ -62,51 +89,13 @@ class Term: # In the future, having IPython channel all its I/O operations through # this class will make it easier to embed it into other environments which # are not a normal terminal (such as a GUI-based shell) - in_s = Stream(sys.stdin,'cin') - out_s = Stream(sys.stdout,'cout') - err_s = Stream(sys.stderr,'cerr') - - # Store the three streams in (err,out,in) order so that if we need to reopen - # them, the error channel is reopened first to provide info. - streams = [err_s,out_s,in_s] - - # The class globals should be the actual 'bare' streams for normal I/O to work - cin = streams[2].stream - cout = streams[1].stream - cerr = streams[0].stream - - def reopen_all(cls): - """Reopen all streams if necessary. - - This should only be called if it is suspected that someting closed - accidentally one of the I/O streams.""" - - any_closed = 0 - - for sn in range(len(cls.streams)): - st = cls.streams[sn] - if st.stream.closed: - any_closed = 1 - new_stream = os.fdopen(os.dup(st.fileno), st.mode,0) - cls.streams[sn] = Stream(new_stream,st.name) - print >> cls.streams[0].stream, \ - '\nWARNING:\nStream Term.%s had to be reopened!' % st.name - - # Rebuild the class globals - cls.cin = cls.streams[2].stream - cls.cout = cls.streams[1].stream - cls.cerr = cls.streams[0].stream - - reopen_all = classmethod(reopen_all) - - def set_stdout(cls,stream): - """Set the stream """ - cls.cout = stream - set_stdout = classmethod(set_stdout) - - def set_stderr(cls,stream): - cls.cerr = stream - set_stderr = classmethod(set_stderr) + def __init__(self,cin=None,cout=None,cerr=None): + self.cin = IOStream(cin,sys.stdin) + self.cout = IOStream(cout,sys.stdout) + self.cerr = IOStream(cerr,sys.stderr) + +# Global variable to be used for all I/O +Term = IOTerm() # Windows-specific code to load Gary Bishop's readline and configure it # automatically for the users @@ -123,8 +112,8 @@ if os.name == 'nt': except AttributeError: pass else: - Term.set_stdout(_out) - Term.set_stderr(_out) + # Remake Term to use the readline i/o facilities + Term = IOTerm(cout=_out,cerr=_out) del _out #**************************************************************************** diff --git a/IPython/iplib.py b/IPython/iplib.py index 434c709..4591960 100644 --- a/IPython/iplib.py +++ b/IPython/iplib.py @@ -6,7 +6,7 @@ Requires Python 2.1 or newer. This file contains all the classes and helper functions specific to IPython. -$Id: iplib.py 602 2005-06-09 03:02:30Z fperez $ +$Id: iplib.py 638 2005-07-18 03:01:41Z fperez $ """ #***************************************************************************** @@ -69,15 +69,6 @@ from IPython.genutils import * # overwrites it (like wx.py.PyShell does) raw_input_original = raw_input -# declares Python 2.2 compatibility symbols: -try: - enumerate -except NameError: - def enumerate(obj): - i = -1 - for item in obj: - i += 1 - yield i, item #**************************************************************************** # Some utility function definitions @@ -1337,7 +1328,7 @@ want to merge them back into the new files.""" % locals() self.interact(header) # Remove locals from namespace - for k in local_ns.keys(): + for k in local_ns: del self.user_ns[k] def interact(self, banner=None): @@ -1363,10 +1354,9 @@ want to merge them back into the new files.""" % locals() # Mark activity in the builtins __builtin__.__dict__['__IPYTHON__active'] += 1 - while 1: - # This is set by a call to %Exit or %Quit - if self.exit_now: - break + + # exit_now is set by a call to %Exit or %Quit + while not self.exit_now: try: if more: prompt = self.outputcache.prompt2 @@ -1421,82 +1411,7 @@ want to merge them back into the new files.""" % locals() "Because of how pdb handles the stack, it is impossible\n" "for IPython to properly format this particular exception.\n" "IPython will resume normal operation.") - except: - # We should never get here except in fairly bizarre situations - # (or b/c of an IPython bug). One reasonable exception is if - # the user sets stdin/out/err to a broken object (or closes - # any of them!) - - fixed_in_out_err = 0 - - # Call the Term I/O class and have it reopen any stream which - # the user might have closed. - Term.reopen_all() - - # Do the same manually for sys.stderr/out/in - - # err first, so we can print at least warnings - if sys.__stderr__.closed: - sys.__stderr__ = os.fdopen(os.dup(2),'w',0) - fixed_err_err = 1 - print >> sys.__stderr__,""" -WARNING: -sys.__stderr__ was closed! -I've tried to reopen it, but bear in mind that things may not work normally -from now. In particular, readline support may have broken. -""" - # Next, check stdin/out - if sys.__stdin__.closed: - sys.__stdin__ = os.fdopen(os.dup(0),'r',0) - fixed_in_out_err = 1 - print >> sys.__stderr__,""" -WARNING: -sys.__stdin__ was closed! -I've tried to reopen it, but bear in mind that things may not work normally -from now. In particular, readline support may have broken. -""" - if sys.__stdout__.closed: - sys.__stdout__ = os.fdopen(os.dup(1),'w',0) - fixed_in_out_err = 1 - print >> sys.__stderr__,""" -WARNING: -sys.__stdout__ was closed! -I've tried to reopen it, but bear in mind that things may not work normally -from now. In particular, readline support may have broken. -""" - - # Now, check mismatch of objects - if sys.stdin is not sys.__stdin__: - sys.stdin = sys.__stdin__ - fixed_in_out_err = 1 - print >> sys.__stderr__,""" -WARNING: -sys.stdin has been reset to sys.__stdin__. -There seemed to be a problem with your sys.stdin. -""" - if sys.stdout is not sys.__stdout__: - sys.stdout = sys.__stdout__ - fixed_in_out_err = 1 - print >> sys.__stderr__,""" -WARNING: -sys.stdout has been reset to sys.__stdout__. -There seemed to be a problem with your sys.stdout. -""" - - if sys.stderr is not sys.__stderr__: - sys.stderr = sys.__stderr__ - fixed_in_out_err = 1 - print >> sys.__stderr__,""" -WARNING: -sys.stderr has been reset to sys.__stderr__. -There seemed to be a problem with your sys.stderr. -""" - # If the problem wasn't a broken out/err, it's an IPython bug - # I wish we could ask the user whether to crash or not, but - # calling any function at this point messes up the stack. - if not fixed_in_out_err: - raise - + # We are off again... __builtin__.__dict__['__IPYTHON__active'] -= 1 diff --git a/IPython/ipmaker.py b/IPython/ipmaker.py index cff6035..c824fe6 100644 --- a/IPython/ipmaker.py +++ b/IPython/ipmaker.py @@ -6,7 +6,7 @@ Requires Python 2.1 or better. This file contains the main make_IPython() starter function. -$Id: ipmaker.py 582 2005-05-13 21:20:00Z fperez $""" +$Id: ipmaker.py 638 2005-07-18 03:01:41Z fperez $""" #***************************************************************************** # Copyright (C) 2001-2004 Fernando Perez. @@ -85,19 +85,8 @@ def make_IPython(argv=None,user_ns=None,debug=1,rc_override=None, IP = shell_class('__IP',user_ns=user_ns,**kw) # Put 'help' in the user namespace - try: - from site import _Helper - except ImportError: - # Use the _Helper class from Python 2.2 for older Python versions - class _Helper: - def __repr__(self): - return "Type help() for interactive help, " \ - "or help(object) for help about object." - def __call__(self, *args, **kwds): - import pydoc - return pydoc.help(*args, **kwds) - else: - IP.user_ns['help'] = _Helper() + from site import _Helper + IP.user_ns['help'] = _Helper() if DEVDEBUG: # For developer debugging only (global flag) diff --git a/doc/ChangeLog b/doc/ChangeLog index 653a3bc..adfcddc 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -1,3 +1,39 @@ +2005-07-17 Fernando Perez + + * IPython/Prompts.py (str_safe): Make unicode-safe. Also remove + some old hacks and clean up a bit other routines; code should be + simpler and a bit faster. + + * IPython/iplib.py (interact): removed some last-resort attempts + to survive broken stdout/stderr. That code was only making it + harder to abstract out the i/o (necessary for gui integration), + and the crashes it could prevent were extremely rare in practice + (besides being fully user-induced in a pretty violent manner). + + * IPython/genutils.py (IOStream.__init__): Simplify the i/o stuff. + Nothing major yet, but the code is simpler to read; this should + make it easier to do more serious modifications in the future. + + * IPython/Extensions/InterpreterExec.py: Fix auto-quoting in pysh, + which broke in .15 (thanks to a report by Ville). + + * IPython/Itpl.py (Itpl.__init__): add unicode support (it may not + be quite correct, I know next to nothing about unicode). This + will allow unicode strings to be used in prompts, amongst other + cases. It also will prevent ipython from crashing when unicode + shows up unexpectedly in many places. If ascii encoding fails, we + assume utf_8. Currently the encoding is not a user-visible + setting, though it could be made so if there is demand for it. + + * IPython/ipmaker.py (make_IPython): remove old 2.1-specific hack. + + * IPython/Struct.py (Struct.merge): switch keys() to iterator. + + * IPython/background_jobs.py: moved 2.2 compatibility to genutils. + + * IPython/genutils.py: Add 2.2 compatibility here, so all other + code can work transparently for 2.2/2.3. + 2005-07-16 Fernando Perez * IPython/ultraTB.py (ExceptionColors): Make a global variable @@ -11,7 +47,7 @@ slightly modified version of the patch in http://www.scipy.net/roundup/ipython/issue34, which also allows me to remove the previous try/except solution (which was costlier). - Thanks to glehmann for the fix. + Thanks to Gaetan Lehmann for the fix. 2005-06-08 Fernando Perez