merge functionality in io and openpy relating to encoding

New functions were introduced in openpy to deal with encoding in
python files. This commit removes redundant code from io and moves
source_to_unicode to openpy.

IPython/core/debugger.py

@@ -1,529 +1,535 b''
 # -*- coding: utf-8 -*-
 """
 Pdb debugger class.
 
 Modified from the standard pdb.Pdb class to avoid including readline, so that
 the command line completion of other programs which include this isn't
 damaged.
 
 In the future, this class will be expanded with improvements over the standard
 pdb.
 
 The code in this file is mainly lifted out of cmd.py in Python 2.2, with minor
 changes. Licensing should therefore be under the standard Python terms.  For
 details on the PSF (Python Software Foundation) standard license, see:
 
 http://www.python.org/2.2.3/license.html"""
 
 #*****************************************************************************
 #
 #       This file is licensed under the PSF license.
 #
 #       Copyright (C) 2001 Python Software Foundation, www.python.org
 #       Copyright (C) 2005-2006 Fernando Perez. <[e-mail]>
 #
 #
 #*****************************************************************************
 from __future__ import print_function
 
 import bdb
 import linecache
 import sys
 
 from IPython.utils import PyColorize
 from IPython.core import ipapi
-from IPython.utils import coloransi, io
+from IPython.utils import coloransi, io, openpy
 from IPython.core.excolors import exception_colors
 
 # See if we can use pydb.
 has_pydb = False
 prompt = 'ipdb> '
 #We have to check this directly from sys.argv, config struct not yet available
 if '--pydb' in sys.argv:
     try:
         import pydb
         if hasattr(pydb.pydb, "runl") and pydb.version>'1.17':
             # Version 1.17 is broken, and that's what ships with Ubuntu Edgy, so we
             # better protect against it.
             has_pydb = True
     except ImportError:
         print("Pydb (http://bashdb.sourceforge.net/pydb/) does not seem to be available")
 
 if has_pydb:
     from pydb import Pdb as OldPdb
     #print "Using pydb for %run -d and post-mortem" #dbg
     prompt = 'ipydb> '
 else:
     from pdb import Pdb as OldPdb
 
 # Allow the set_trace code to operate outside of an ipython instance, even if
 # it does so with some limitations.  The rest of this support is implemented in
 # the Tracer constructor.
 def BdbQuit_excepthook(et,ev,tb):
     if et==bdb.BdbQuit:
         print('Exiting Debugger.')
     else:
         BdbQuit_excepthook.excepthook_ori(et,ev,tb)
 
 def BdbQuit_IPython_excepthook(self,et,ev,tb,tb_offset=None):
     print('Exiting Debugger.')
 
 
 class Tracer(object):
     """Class for local debugging, similar to pdb.set_trace.
 
     Instances of this class, when called, behave like pdb.set_trace, but
     providing IPython's enhanced capabilities.
 
     This is implemented as a class which must be initialized in your own code
     and not as a standalone function because we need to detect at runtime
     whether IPython is already active or not.  That detection is done in the
     constructor, ensuring that this code plays nicely with a running IPython,
     while functioning acceptably (though with limitations) if outside of it.
     """
 
     def __init__(self,colors=None):
         """Create a local debugger instance.
 
         :Parameters:
 
           - `colors` (None): a string containing the name of the color scheme to
         use, it must be one of IPython's valid color schemes.  If not given, the
         function will default to the current IPython scheme when running inside
         IPython, and to 'NoColor' otherwise.
 
         Usage example:
 
         from IPython.core.debugger import Tracer; debug_here = Tracer()
 
         ... later in your code
         debug_here()  # -> will open up the debugger at that point.
 
         Once the debugger activates, you can use all of its regular commands to
         step through code, set breakpoints, etc.  See the pdb documentation
         from the Python standard library for usage details.
         """
 
         try:
             ip = get_ipython()
         except NameError:
             # Outside of ipython, we set our own exception hook manually
             BdbQuit_excepthook.excepthook_ori = sys.excepthook
             sys.excepthook = BdbQuit_excepthook
             def_colors = 'NoColor'
             try:
                 # Limited tab completion support
                 import readline
                 readline.parse_and_bind('tab: complete')
             except ImportError:
                 pass
         else:
             # In ipython, we use its custom exception handler mechanism
             def_colors = ip.colors
             ip.set_custom_exc((bdb.BdbQuit,), BdbQuit_IPython_excepthook)
 
         if colors is None:
             colors = def_colors
 
         # The stdlib debugger internally uses a modified repr from the `repr`
         # module, that limits the length of printed strings to a hardcoded
         # limit of 30 characters.  That much trimming is too aggressive, let's
         # at least raise that limit to 80 chars, which should be enough for
         # most interactive uses.
         try:
             from repr import aRepr
             aRepr.maxstring = 80
         except:
             # This is only a user-facing convenience, so any error we encounter
             # here can be warned about but can be otherwise ignored.  These
             # printouts will tell us about problems if this API changes 
             import traceback
             traceback.print_exc()
 
         self.debugger = Pdb(colors)
 
     def __call__(self):
         """Starts an interactive debugger at the point where called.
 
         This is similar to the pdb.set_trace() function from the std lib, but
         using IPython's enhanced debugger."""
 
         self.debugger.set_trace(sys._getframe().f_back)
 
 
 def decorate_fn_with_doc(new_fn, old_fn, additional_text=""):
     """Make new_fn have old_fn's doc string. This is particularly useful
     for the do_... commands that hook into the help system.
     Adapted from from a comp.lang.python posting
     by Duncan Booth."""
     def wrapper(*args, **kw):
         return new_fn(*args, **kw)
     if old_fn.__doc__:
         wrapper.__doc__ = old_fn.__doc__ + additional_text
     return wrapper
 
 
 def _file_lines(fname):
     """Return the contents of a named file as a list of lines.
 
     This function never raises an IOError exception: if the file can't be
     read, it simply returns an empty list."""
 
     try:
         outfile = open(fname)
     except IOError:
         return []
     else:
         out = outfile.readlines()
         outfile.close()
         return out
 
 
 class Pdb(OldPdb):
     """Modified Pdb class, does not load readline."""
 
     def __init__(self,color_scheme='NoColor',completekey=None,
                  stdin=None, stdout=None):
 
         # Parent constructor:
         if has_pydb and completekey is None:
             OldPdb.__init__(self,stdin=stdin,stdout=io.stdout)
         else:
             OldPdb.__init__(self,completekey,stdin,stdout)
 
         self.prompt = prompt # The default prompt is '(Pdb)'
 
         # IPython changes...
         self.is_pydb = has_pydb
 
         self.shell = ipapi.get()
 
         if self.is_pydb:
 
             # interactiveshell.py's ipalias seems to want pdb's checkline
             # which located in pydb.fn
             import pydb.fns
             self.checkline = lambda filename, lineno: \
                              pydb.fns.checkline(self, filename, lineno)
 
             self.curframe = None
             self.do_restart = self.new_do_restart
 
             self.old_all_completions = self.shell.Completer.all_completions
             self.shell.Completer.all_completions=self.all_completions
 
             self.do_list = decorate_fn_with_doc(self.list_command_pydb,
                                                 OldPdb.do_list)
             self.do_l     = self.do_list
             self.do_frame = decorate_fn_with_doc(self.new_do_frame,
                                                  OldPdb.do_frame)
 
         self.aliases = {}
 
         # Create color table: we copy the default one from the traceback
         # module and add a few attributes needed for debugging
         self.color_scheme_table = exception_colors()
 
         # shorthands
         C = coloransi.TermColors
         cst = self.color_scheme_table
 
         cst['NoColor'].colors.breakpoint_enabled = C.NoColor
         cst['NoColor'].colors.breakpoint_disabled = C.NoColor
 
         cst['Linux'].colors.breakpoint_enabled = C.LightRed
         cst['Linux'].colors.breakpoint_disabled = C.Red
 
         cst['LightBG'].colors.breakpoint_enabled = C.LightRed
         cst['LightBG'].colors.breakpoint_disabled = C.Red
 
         self.set_colors(color_scheme)
 
         # Add a python parser so we can syntax highlight source while
         # debugging.
         self.parser = PyColorize.Parser()
 
     def set_colors(self, scheme):
         """Shorthand access to the color table scheme selector method."""
         self.color_scheme_table.set_active_scheme(scheme)
 
     def interaction(self, frame, traceback):
         self.shell.set_completer_frame(frame)
         OldPdb.interaction(self, frame, traceback)
 
     def new_do_up(self, arg):
         OldPdb.do_up(self, arg)
         self.shell.set_completer_frame(self.curframe)
     do_u = do_up = decorate_fn_with_doc(new_do_up, OldPdb.do_up)
 
     def new_do_down(self, arg):
         OldPdb.do_down(self, arg)
         self.shell.set_completer_frame(self.curframe)
 
     do_d = do_down = decorate_fn_with_doc(new_do_down, OldPdb.do_down)
 
     def new_do_frame(self, arg):
         OldPdb.do_frame(self, arg)
         self.shell.set_completer_frame(self.curframe)
 
     def new_do_quit(self, arg):
 
         if hasattr(self, 'old_all_completions'):
             self.shell.Completer.all_completions=self.old_all_completions
 
 
         return OldPdb.do_quit(self, arg)
 
     do_q = do_quit = decorate_fn_with_doc(new_do_quit, OldPdb.do_quit)
 
     def new_do_restart(self, arg):
         """Restart command. In the context of ipython this is exactly the same
         thing as 'quit'."""
         self.msg("Restart doesn't make sense here. Using 'quit' instead.")
         return self.do_quit(arg)
 
     def postloop(self):
         self.shell.set_completer_frame(None)
 
     def print_stack_trace(self):
         try:
             for frame_lineno in self.stack:
                 self.print_stack_entry(frame_lineno, context = 5)
         except KeyboardInterrupt:
             pass
 
     def print_stack_entry(self,frame_lineno,prompt_prefix='\n-> ',
                           context = 3):
         #frame, lineno = frame_lineno
         print(self.format_stack_entry(frame_lineno, '', context), file=io.stdout)
 
         # vds: >>
         frame, lineno = frame_lineno
         filename = frame.f_code.co_filename
         self.shell.hooks.synchronize_with_editor(filename, lineno, 0)
         # vds: <<
 
     def format_stack_entry(self, frame_lineno, lprefix=': ', context = 3):
         import linecache, repr
 
         ret = []
 
         Colors = self.color_scheme_table.active_colors
         ColorsNormal = Colors.Normal
         tpl_link = '%s%%s%s' % (Colors.filenameEm, ColorsNormal)
         tpl_call = '%s%%s%s%%s%s' % (Colors.vName, Colors.valEm, ColorsNormal)
         tpl_line = '%%s%s%%s %s%%s' % (Colors.lineno, ColorsNormal)
         tpl_line_em = '%%s%s%%s %s%%s%s' % (Colors.linenoEm, Colors.line,
                                             ColorsNormal)
 
         frame, lineno = frame_lineno
 
         return_value = ''
         if '__return__' in frame.f_locals:
             rv = frame.f_locals['__return__']
             #return_value += '->'
             return_value += repr.repr(rv) + '\n'
         ret.append(return_value)
 
         #s = filename + '(' + `lineno` + ')'
         filename = self.canonic(frame.f_code.co_filename)
         link = tpl_link % filename
 
         if frame.f_code.co_name:
             func = frame.f_code.co_name
         else:
             func = "<lambda>"
 
         call = ''
         if func != '?':
             if '__args__' in frame.f_locals:
                 args = repr.repr(frame.f_locals['__args__'])
             else:
                 args = '()'
             call = tpl_call % (func, args)
 
         # The level info should be generated in the same format pdb uses, to
         # avoid breaking the pdbtrack functionality of python-mode in *emacs.
         if frame is self.curframe:
             ret.append('> ')
         else:
             ret.append('  ')
         ret.append('%s(%s)%s\n' % (link,lineno,call))
 
         start = lineno - 1 - context//2
         lines = linecache.getlines(filename)
-        encoding = io.guess_encoding(lines)
+        try:
+            encoding, _ = openpy.detect_encoding(lambda :lines[:2].pop(0))
+        except SyntaxError:
+            encoding = "ascii"
         start = max(start, 0)
         start = min(start, len(lines) - context)
         lines = lines[start : start + context]
 
         for i,line in enumerate(lines):
             show_arrow = (start + 1 + i == lineno)
             linetpl = (frame is self.curframe or show_arrow) \
                       and tpl_line_em \
                       or tpl_line
             ret.append(self.__format_line(linetpl, filename,
-                                          start + 1 + i, line.decode(encoding),
+                                          start + 1 + i, line.decode(encoding, errors="replace"),
                                           arrow = show_arrow) )
         return ''.join(ret)
 
     def __format_line(self, tpl_line, filename, lineno, line, arrow = False):
         bp_mark = ""
         bp_mark_color = ""
 
         scheme = self.color_scheme_table.active_scheme_name
         new_line, err = self.parser.format2(line, 'str', scheme)
         if not err: line = new_line
 
         bp = None
         if lineno in self.get_file_breaks(filename):
             bps = self.get_breaks(filename, lineno)
             bp = bps[-1]
 
         if bp:
             Colors = self.color_scheme_table.active_colors
             bp_mark = str(bp.number)
             bp_mark_color = Colors.breakpoint_enabled
             if not bp.enabled:
                 bp_mark_color = Colors.breakpoint_disabled
 
         numbers_width = 7
         if arrow:
             # This is the line with the error
             pad = numbers_width - len(str(lineno)) - len(bp_mark)
             if pad >= 3:
                 marker = '-'*(pad-3) + '-> '
             elif pad == 2:
                  marker = '> '
             elif pad == 1:
                  marker = '>'
             else:
                  marker = ''
             num = '%s%s' % (marker, str(lineno))
             line = tpl_line % (bp_mark_color + bp_mark, num, line)
         else:
             num = '%*s' % (numbers_width - len(bp_mark), str(lineno))
             line = tpl_line % (bp_mark_color + bp_mark, num, line)
 
         return line
 
     def list_command_pydb(self, arg):
         """List command to use if we have a newer pydb installed"""
         filename, first, last = OldPdb.parse_list_cmd(self, arg)
         if filename is not None:
             self.print_list_lines(filename, first, last)
 
     def print_list_lines(self, filename, first, last):
         """The printing (as opposed to the parsing part of a 'list'
         command."""
         try:
             Colors = self.color_scheme_table.active_colors
             ColorsNormal = Colors.Normal
             tpl_line = '%%s%s%%s %s%%s' % (Colors.lineno, ColorsNormal)
             tpl_line_em = '%%s%s%%s %s%%s%s' % (Colors.linenoEm, Colors.line, ColorsNormal)
             src = []
             lines = linecache.getlines(filename)
-            encoding = io.guess_encoding(lines)
+            try:
+                encoding, _ = openpy.detect_encoding(lambda :lines[:2].pop(0))
+            except SyntaxError:
+                encoding = "ascii"
             for lineno in range(first, last+1):
-                line = lines[lineno].decode(encoding)
+                line = lines[lineno].decode(encoding, errors="replace")
                 if not line:
                     break
 
                 if lineno == self.curframe.f_lineno:
                     line = self.__format_line(tpl_line_em, filename, lineno, line, arrow = True)
                 else:
                     line = self.__format_line(tpl_line, filename, lineno, line, arrow = False)
 
                 src.append(line)
                 self.lineno = lineno
 
             print(''.join(src), file=io.stdout)
 
         except KeyboardInterrupt:
             pass
 
     def do_list(self, arg):
         self.lastcmd = 'list'
         last = None
         if arg:
             try:
                 x = eval(arg, {}, {})
                 if type(x) == type(()):
                     first, last = x
                     first = int(first)
                     last = int(last)
                     if last < first:
                         # Assume it's a count
                         last = first + last
                 else:
                     first = max(1, int(x) - 5)
             except:
                 print('*** Error in argument:', repr(arg))
                 return
         elif self.lineno is None:
             first = max(1, self.curframe.f_lineno - 5)
         else:
             first = self.lineno + 1
         if last is None:
             last = first + 10
         self.print_list_lines(self.curframe.f_code.co_filename, first, last)
 
         # vds: >>
         lineno = first
         filename = self.curframe.f_code.co_filename
         self.shell.hooks.synchronize_with_editor(filename, lineno, 0)
         # vds: <<
 
     do_l = do_list
 
     def do_pdef(self, arg):
         """The debugger interface to magic_pdef"""
         namespaces = [('Locals', self.curframe.f_locals),
                       ('Globals', self.curframe.f_globals)]
         self.shell.find_line_magic('pdef')(arg, namespaces=namespaces)
 
     def do_pdoc(self, arg):
         """The debugger interface to magic_pdoc"""
         namespaces = [('Locals', self.curframe.f_locals),
                       ('Globals', self.curframe.f_globals)]
         self.shell.find_line_magic('pdoc')(arg, namespaces=namespaces)
 
     def do_pinfo(self, arg):
         """The debugger equivalant of ?obj"""
         namespaces = [('Locals', self.curframe.f_locals),
                       ('Globals', self.curframe.f_globals)]
         self.shell.find_line_magic('pinfo')("pinfo %s" % arg,
                                             namespaces=namespaces)
 
     def checkline(self, filename, lineno):
         """Check whether specified line seems to be executable.
 
         Return `lineno` if it is, 0 if not (e.g. a docstring, comment, blank
         line or EOF). Warning: testing is not comprehensive.
         """
         #######################################################################
         # XXX Hack!  Use python-2.5 compatible code for this call, because with
         # all of our changes, we've drifted from the pdb api in 2.6.  For now,
         # changing:
         #
         #line = linecache.getline(filename, lineno, self.curframe.f_globals)
         # to:
         #
         line = linecache.getline(filename, lineno)
         #
         # does the trick.  But in reality, we need to fix this by reconciling
         # our updates with the new Pdb APIs in Python 2.6.
         #
         # End hack.  The rest of this method is copied verbatim from 2.6 pdb.py
         #######################################################################
 
         if not line:
             print('End of file', file=self.stdout)
             return 0
         line = line.strip()
         # Don't allow setting breakpoint at a blank line
         if (not line or (line[0] == '#') or
              (line[:3] == '"""') or line[:3] == "'''"):
             print('*** Blank or comment', file=self.stdout)
             return 0
         return lineno

IPython/core/oinspect.py

@@ -1,842 +1,843 b''
 # -*- coding: utf-8 -*-
 """Tools for inspecting Python objects.
 
 Uses syntax highlighting for presenting the various information elements.
 
 Similar in spirit to the inspect module, but all calls take a name argument to
 reference the name under which an object is being read.
 """
 
 #*****************************************************************************
 #       Copyright (C) 2001-2004 Fernando Perez <[e-mail]>
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #*****************************************************************************
 from __future__ import print_function
 
 __all__ = ['Inspector','InspectColors']
 
 # stdlib modules
 import __builtin__
 import inspect
 import linecache
 import os
 import sys
 import types
 from collections import namedtuple
 try:
     from itertools import izip_longest
 except ImportError:
     from itertools import zip_longest as izip_longest
 
 # IPython's own
 from IPython.core import page
 from IPython.testing.skipdoctest import skip_doctest_py3
 from IPython.utils import PyColorize
 from IPython.utils import io
+from IPython.utils import openpy
 from IPython.utils import py3compat
 from IPython.utils.text import indent
 from IPython.utils.wildcard import list_namespace
 from IPython.utils.coloransi import *
 
 #****************************************************************************
 # Builtin color schemes
 
 Colors = TermColors  # just a shorthand
 
 # Build a few color schemes
 NoColor = ColorScheme(
     'NoColor',{
     'header' : Colors.NoColor,
     'normal' : Colors.NoColor  # color off (usu. Colors.Normal)
     }  )
 
 LinuxColors = ColorScheme(
     'Linux',{
     'header' : Colors.LightRed,
     'normal' : Colors.Normal  # color off (usu. Colors.Normal)
     } )
 
 LightBGColors = ColorScheme(
     'LightBG',{
     'header' : Colors.Red,
     'normal' : Colors.Normal  # color off (usu. Colors.Normal)
     }  )
 
 # Build table of color schemes (needed by the parser)
 InspectColors = ColorSchemeTable([NoColor,LinuxColors,LightBGColors],
                                  'Linux')
 
 #****************************************************************************
 # Auxiliary functions and objects
 
 # See the messaging spec for the definition of all these fields.  This list
 # effectively defines the order of display
 info_fields = ['type_name', 'base_class', 'string_form', 'namespace',
                'length', 'file', 'definition', 'docstring', 'source',
                'init_definition', 'class_docstring', 'init_docstring',
                'call_def', 'call_docstring',
                # These won't be printed but will be used to determine how to
                # format the object
                'ismagic', 'isalias', 'isclass', 'argspec', 'found', 'name'
                ]
 
 
 def object_info(**kw):
     """Make an object info dict with all fields present."""
     infodict = dict(izip_longest(info_fields, [None]))
     infodict.update(kw)
     return infodict
 
 
 def getdoc(obj):
     """Stable wrapper around inspect.getdoc.
 
     This can't crash because of attribute problems.
 
     It also attempts to call a getdoc() method on the given object.  This
     allows objects which provide their docstrings via non-standard mechanisms
     (like Pyro proxies) to still be inspected by ipython's ? system."""
     # Allow objects to offer customized documentation via a getdoc method:
     try:
         ds = obj.getdoc()
     except Exception:
         pass
     else:
         # if we get extra info, we add it to the normal docstring.
         if isinstance(ds, basestring):
             return inspect.cleandoc(ds)
     
     try:
         return inspect.getdoc(obj)
     except Exception:
         # Harden against an inspect failure, which can occur with
         # SWIG-wrapped extensions.
         return None
 
 
 def getsource(obj,is_binary=False):
     """Wrapper around inspect.getsource.
 
     This can be modified by other projects to provide customized source
     extraction.
 
     Inputs:
 
     - obj: an object whose source code we will attempt to extract.
 
     Optional inputs:
 
     - is_binary: whether the object is known to come from a binary source.
     This implementation will skip returning any output for binary objects, but
     custom extractors may know how to meaningfully process them."""
 
     if is_binary:
         return None
     else:
         # get source if obj was decorated with @decorator
         if hasattr(obj,"__wrapped__"):
             obj = obj.__wrapped__
         try:
             src = inspect.getsource(obj)
         except TypeError:
             if hasattr(obj,'__class__'):
                 src = inspect.getsource(obj.__class__)
         return src
 
 def getargspec(obj):
     """Get the names and default values of a function's arguments.
 
     A tuple of four things is returned: (args, varargs, varkw, defaults).
     'args' is a list of the argument names (it may contain nested lists).
     'varargs' and 'varkw' are the names of the * and ** arguments or None.
     'defaults' is an n-tuple of the default values of the last n arguments.
 
     Modified version of inspect.getargspec from the Python Standard
     Library."""
 
     if inspect.isfunction(obj):
         func_obj = obj
     elif inspect.ismethod(obj):
         func_obj = obj.im_func
     elif hasattr(obj, '__call__'):
         func_obj = obj.__call__
     else:
         raise TypeError('arg is not a Python function')
     args, varargs, varkw = inspect.getargs(func_obj.func_code)
     return args, varargs, varkw, func_obj.func_defaults
 
 
 def format_argspec(argspec):
     """Format argspect, convenience wrapper around inspect's.
 
     This takes a dict instead of ordered arguments and calls
     inspect.format_argspec with the arguments in the necessary order.
     """
     return inspect.formatargspec(argspec['args'], argspec['varargs'],
                                  argspec['varkw'], argspec['defaults'])
 
 
 def call_tip(oinfo, format_call=True):
     """Extract call tip data from an oinfo dict.
 
     Parameters
     ----------
     oinfo : dict
 
     format_call : bool, optional
       If True, the call line is formatted and returned as a string.  If not, a
       tuple of (name, argspec) is returned.
 
     Returns
     -------
     call_info : None, str or (str, dict) tuple.
       When format_call is True, the whole call information is formattted as a
       single string.  Otherwise, the object's name and its argspec dict are
       returned.  If no call information is available, None is returned.
 
     docstring : str or None
       The most relevant docstring for calling purposes is returned, if
       available.  The priority is: call docstring for callable instances, then
       constructor docstring for classes, then main object's docstring otherwise
       (regular functions).
     """
     # Get call definition
     argspec = oinfo.get('argspec')
     if argspec is None:
         call_line = None
     else:
         # Callable objects will have 'self' as their first argument, prune
         # it out if it's there for clarity (since users do *not* pass an
         # extra first argument explicitly).
         try:
             has_self = argspec['args'][0] == 'self'
         except (KeyError, IndexError):
             pass
         else:
             if has_self:
                 argspec['args'] = argspec['args'][1:]
 
         call_line = oinfo['name']+format_argspec(argspec)
 
     # Now get docstring.
     # The priority is: call docstring, constructor docstring, main one.
     doc = oinfo.get('call_docstring')
     if doc is None:
         doc = oinfo.get('init_docstring')
     if doc is None:
         doc = oinfo.get('docstring','')
 
     return call_line, doc
 
 
 def find_file(obj):
     """Find the absolute path to the file where an object was defined.
 
     This is essentially a robust wrapper around `inspect.getabsfile`.
 
     Returns None if no file can be found.
 
     Parameters
     ----------
     obj : any Python object
 
     Returns
     -------
     fname : str
       The absolute path to the file where the object was defined.
     """
     # get source if obj was decorated with @decorator
     if hasattr(obj, '__wrapped__'):
         obj = obj.__wrapped__
 
     fname = None
     try:
         fname = inspect.getabsfile(obj)
     except TypeError:
         # For an instance, the file that matters is where its class was
         # declared.
         if hasattr(obj, '__class__'):
             try:
                 fname = inspect.getabsfile(obj.__class__)
             except TypeError:
                 # Can happen for builtins
                 pass
     except:
         pass
     return fname
 
 
 def find_source_lines(obj):
     """Find the line number in a file where an object was defined.
 
     This is essentially a robust wrapper around `inspect.getsourcelines`.
 
     Returns None if no file can be found.
 
     Parameters
     ----------
     obj : any Python object
 
     Returns
     -------
     lineno : int
       The line number where the object definition starts.
     """
     # get source if obj was decorated with @decorator
     if hasattr(obj, '__wrapped__'):
         obj = obj.__wrapped__
     
     try:
         try:
             lineno = inspect.getsourcelines(obj)[1]
         except TypeError:
             # For instances, try the class object like getsource() does
             if hasattr(obj, '__class__'):
                 lineno = inspect.getsourcelines(obj.__class__)[1]
     except:
         return None
 
     return lineno
 
 
 class Inspector:
     def __init__(self, color_table=InspectColors,
                  code_color_table=PyColorize.ANSICodeColors,
                  scheme='NoColor',
                  str_detail_level=0):
         self.color_table = color_table
         self.parser = PyColorize.Parser(code_color_table,out='str')
         self.format = self.parser.format
         self.str_detail_level = str_detail_level
         self.set_active_scheme(scheme)
 
     def _getdef(self,obj,oname=''):
         """Return the definition header for any callable object.
 
         If any exception is generated, None is returned instead and the
         exception is suppressed."""
 
         try:
             # We need a plain string here, NOT unicode!
             hdef = oname + inspect.formatargspec(*getargspec(obj))
             return py3compat.unicode_to_str(hdef, 'ascii')
         except:
             return None
 
     def __head(self,h):
         """Return a header string with proper colors."""
         return '%s%s%s' % (self.color_table.active_colors.header,h,
                            self.color_table.active_colors.normal)
 
     def set_active_scheme(self, scheme):
         self.color_table.set_active_scheme(scheme)
         self.parser.color_table.set_active_scheme(scheme)
 
     def noinfo(self, msg, oname):
         """Generic message when no information is found."""
         print('No %s found' % msg, end=' ')
         if oname:
             print('for %s' % oname)
         else:
             print()
 
     def pdef(self, obj, oname=''):
         """Print the definition header for any callable object.
 
         If the object is a class, print the constructor information."""
 
         if not callable(obj):
             print('Object is not callable.')
             return
 
         header = ''
 
         if inspect.isclass(obj):
             header = self.__head('Class constructor information:\n')
             obj = obj.__init__
         elif (not py3compat.PY3) and type(obj) is types.InstanceType:
             obj = obj.__call__
 
         output = self._getdef(obj,oname)
         if output is None:
             self.noinfo('definition header',oname)
         else:
             print(header,self.format(output), end=' ', file=io.stdout)
 
     # In Python 3, all classes are new-style, so they all have __init__.
     @skip_doctest_py3
     def pdoc(self,obj,oname='',formatter = None):
         """Print the docstring for any object.
 
         Optional:
         -formatter: a function to run the docstring through for specially
         formatted docstrings.
 
         Examples
         --------
 
         In [1]: class NoInit:
            ...:     pass
 
         In [2]: class NoDoc:
            ...:     def __init__(self):
            ...:         pass
 
         In [3]: %pdoc NoDoc
         No documentation found for NoDoc
 
         In [4]: %pdoc NoInit
         No documentation found for NoInit
 
         In [5]: obj = NoInit()
 
         In [6]: %pdoc obj
         No documentation found for obj
 
         In [5]: obj2 = NoDoc()
 
         In [6]: %pdoc obj2
         No documentation found for obj2
         """
 
         head = self.__head  # For convenience
         lines = []
         ds = getdoc(obj)
         if formatter:
             ds = formatter(ds)
         if ds:
             lines.append(head("Class Docstring:"))
             lines.append(indent(ds))
         if inspect.isclass(obj) and hasattr(obj, '__init__'):
             init_ds = getdoc(obj.__init__)
             if init_ds is not None:
                 lines.append(head("Constructor Docstring:"))
                 lines.append(indent(init_ds))
         elif hasattr(obj,'__call__'):
             call_ds = getdoc(obj.__call__)
             if call_ds:
                 lines.append(head("Calling Docstring:"))
                 lines.append(indent(call_ds))
 
         if not lines:
             self.noinfo('documentation',oname)
         else:
             page.page('\n'.join(lines))
 
     def psource(self,obj,oname=''):
         """Print the source code for an object."""
 
         # Flush the source cache because inspect can return out-of-date source
         linecache.checkcache()
         try:
             src = getsource(obj)
         except:
             self.noinfo('source',oname)
         else:
             page.page(self.format(py3compat.unicode_to_str(src)))
 
     def pfile(self, obj, oname=''):
         """Show the whole file where an object was defined."""
         
         lineno = find_source_lines(obj)
         if lineno is None:
             self.noinfo('file', oname)
             return
 
         ofile = find_file(obj)
         # run contents of file through pager starting at line where the object
         # is defined, as long as the file isn't binary and is actually on the
         # filesystem.
         if ofile.endswith(('.so', '.dll', '.pyd')):
             print('File %r is binary, not printing.' % ofile)
         elif not os.path.isfile(ofile):
             print('File %r does not exist, not printing.' % ofile)
         else:
             # Print only text files, not extension binaries.  Note that
             # getsourcelines returns lineno with 1-offset and page() uses
             # 0-offset, so we must adjust.
-            page.page(self.format(io.source_to_unicode(open(ofile).read())), lineno-1)
+            page.page(self.format(openpy.read_py_file(ofile, skip_encoding_cookie=False)), lineno - 1)
 
     def _format_fields(self, fields, title_width=12):
         """Formats a list of fields for display.
 
         Parameters
         ----------
         fields : list
           A list of 2-tuples: (field_title, field_content)
         title_width : int
           How many characters to pad titles to. Default 12.
         """
         out = []
         header = self.__head
         for title, content in fields:
             if len(content.splitlines()) > 1:
                 title = header(title + ":") + "\n"
             else:
                 title = header((title+":").ljust(title_width))
             out.append(title + content)
         return "\n".join(out)
 
     # The fields to be displayed by pinfo: (fancy_name, key_in_info_dict)
     pinfo_fields1 = [("Type", "type_name"),
                     ]
                     
     pinfo_fields2 = [("String Form", "string_form"),
                     ]
 
     pinfo_fields3 = [("Length", "length"),
                     ("File", "file"),
                     ("Definition", "definition"),
                     ]
 
     pinfo_fields_obj = [("Class Docstring", "class_docstring"),
                         ("Constructor Docstring","init_docstring"),
                         ("Call def", "call_def"),
                         ("Call docstring", "call_docstring")]
 
     def pinfo(self,obj,oname='',formatter=None,info=None,detail_level=0):
         """Show detailed information about an object.
 
         Optional arguments:
 
         - oname: name of the variable pointing to the object.
 
         - formatter: special formatter for docstrings (see pdoc)
 
         - info: a structure with some information fields which may have been
         precomputed already.
 
         - detail_level: if set to 1, more information is given.
         """
         info = self.info(obj, oname=oname, formatter=formatter,
                             info=info, detail_level=detail_level)
         displayfields = []
         def add_fields(fields):
             for title, key in fields:
                 field = info[key]
                 if field is not None:
                     displayfields.append((title, field.rstrip()))
         
         add_fields(self.pinfo_fields1)
         
         # Base class for old-style instances
         if (not py3compat.PY3) and isinstance(obj, types.InstanceType) and info['base_class']:
             displayfields.append(("Base Class", info['base_class'].rstrip()))
         
         add_fields(self.pinfo_fields2)
         
         # Namespace
         if info['namespace'] != 'Interactive':
             displayfields.append(("Namespace", info['namespace'].rstrip()))
 
         add_fields(self.pinfo_fields3)
         
         # Source or docstring, depending on detail level and whether
         # source found.
         if detail_level > 0 and info['source'] is not None:
             displayfields.append(("Source", self.format(py3compat.cast_bytes_py2(info['source']))))
         elif info['docstring'] is not None:
             displayfields.append(("Docstring", info["docstring"]))
 
         # Constructor info for classes
         if info['isclass']:
             if info['init_definition'] or info['init_docstring']:
                 displayfields.append(("Constructor information", ""))
                 if info['init_definition'] is not None:
                     displayfields.append((" Definition",
                                     info['init_definition'].rstrip()))
                 if info['init_docstring'] is not None:
                     displayfields.append((" Docstring",
                                         indent(info['init_docstring'])))
 
         # Info for objects:
         else:
             add_fields(self.pinfo_fields_obj)
 
         # Finally send to printer/pager:
         if displayfields:
             page.page(self._format_fields(displayfields))
 
     def info(self, obj, oname='', formatter=None, info=None, detail_level=0):
         """Compute a dict with detailed information about an object.
 
         Optional arguments:
 
         - oname: name of the variable pointing to the object.
 
         - formatter: special formatter for docstrings (see pdoc)
 
         - info: a structure with some information fields which may have been
         precomputed already.
 
         - detail_level: if set to 1, more information is given.
         """
 
         obj_type = type(obj)
 
         header = self.__head
         if info is None:
             ismagic = 0
             isalias = 0
             ospace = ''
         else:
             ismagic = info.ismagic
             isalias = info.isalias
             ospace = info.namespace
 
         # Get docstring, special-casing aliases:
         if isalias:
             if not callable(obj):
                 try:
                     ds = "Alias to the system command:\n  %s" % obj[1]
                 except:
                     ds = "Alias: " + str(obj)
             else:
                 ds = "Alias to " + str(obj)
                 if obj.__doc__:
                     ds += "\nDocstring:\n" + obj.__doc__
         else:
             ds = getdoc(obj)
             if ds is None:
                 ds = '<no docstring>'
         if formatter is not None:
             ds = formatter(ds)
 
         # store output in a dict, we initialize it here and fill it as we go
         out = dict(name=oname, found=True, isalias=isalias, ismagic=ismagic)
 
         string_max = 200 # max size of strings to show (snipped if longer)
         shalf = int((string_max -5)/2)
 
         if ismagic:
             obj_type_name = 'Magic function'
         elif isalias:
             obj_type_name = 'System alias'
         else:
             obj_type_name = obj_type.__name__
         out['type_name'] = obj_type_name
 
         try:
             bclass = obj.__class__
             out['base_class'] = str(bclass)
         except: pass
 
         # String form, but snip if too long in ? form (full in ??)
         if detail_level >= self.str_detail_level:
             try:
                 ostr = str(obj)
                 str_head = 'string_form'
                 if not detail_level and len(ostr)>string_max:
                     ostr = ostr[:shalf] + ' <...> ' + ostr[-shalf:]
                     ostr = ("\n" + " " * len(str_head.expandtabs())).\
                             join(q.strip() for q in ostr.split("\n"))
                 out[str_head] = ostr
             except:
                 pass
 
         if ospace:
             out['namespace'] = ospace
 
         # Length (for strings and lists)
         try:
             out['length'] = str(len(obj))
         except: pass
 
         # Filename where object was defined
         binary_file = False
         fname = find_file(obj)
         if fname is None:
             # if anything goes wrong, we don't want to show source, so it's as
             # if the file was binary
             binary_file = True
         else:
             if fname.endswith(('.so', '.dll', '.pyd')):
                 binary_file = True
             elif fname.endswith('<string>'):
                 fname = 'Dynamically generated function. No source code available.'
             out['file'] = fname
         
         # reconstruct the function definition and print it:
         defln = self._getdef(obj, oname)
         if defln:
             out['definition'] = self.format(defln)
 
         # Docstrings only in detail 0 mode, since source contains them (we
         # avoid repetitions).  If source fails, we add them back, see below.
         if ds and detail_level == 0:
                 out['docstring'] = ds
 
         # Original source code for any callable
         if detail_level:
             # Flush the source cache because inspect can return out-of-date
             # source
             linecache.checkcache()
             source = None
             try:
                 try:
                     source = getsource(obj, binary_file)
                 except TypeError:
                     if hasattr(obj, '__class__'):
                         source = getsource(obj.__class__, binary_file)
                 if source is not None:
                     out['source'] = source.rstrip()
             except Exception:
                 pass
 
             if ds and source is None:
                 out['docstring'] = ds
 
 
         # Constructor docstring for classes
         if inspect.isclass(obj):
             out['isclass'] = True
             # reconstruct the function definition and print it:
             try:
                 obj_init =  obj.__init__
             except AttributeError:
                 init_def = init_ds = None
             else:
                 init_def = self._getdef(obj_init,oname)
                 init_ds  = getdoc(obj_init)
                 # Skip Python's auto-generated docstrings
                 if init_ds and \
                        init_ds.startswith('x.__init__(...) initializes'):
                     init_ds = None
 
             if init_def or init_ds:
                 if init_def:
                     out['init_definition'] = self.format(init_def)
                 if init_ds:
                     out['init_docstring'] = init_ds
 
         # and class docstring for instances:
         else:
             # First, check whether the instance docstring is identical to the
             # class one, and print it separately if they don't coincide.  In
             # most cases they will, but it's nice to print all the info for
             # objects which use instance-customized docstrings.
             if ds:
                 try:
                     cls = getattr(obj,'__class__')
                 except:
                     class_ds = None
                 else:
                     class_ds = getdoc(cls)
                 # Skip Python's auto-generated docstrings
                 if class_ds and \
                        (class_ds.startswith('function(code, globals[,') or \
                    class_ds.startswith('instancemethod(function, instance,') or \
                    class_ds.startswith('module(name[,') ):
                     class_ds = None
                 if class_ds and ds != class_ds:
                     out['class_docstring'] = class_ds
 
             # Next, try to show constructor docstrings
             try:
                 init_ds = getdoc(obj.__init__)
                 # Skip Python's auto-generated docstrings
                 if init_ds and \
                        init_ds.startswith('x.__init__(...) initializes'):
                     init_ds = None
             except AttributeError:
                 init_ds = None
             if init_ds:
                 out['init_docstring'] = init_ds
 
             # Call form docstring for callable instances
             if hasattr(obj, '__call__'):
                 call_def = self._getdef(obj.__call__, oname)
                 if call_def is not None:
                     out['call_def'] = self.format(call_def)
                 call_ds = getdoc(obj.__call__)
                 # Skip Python's auto-generated docstrings
                 if call_ds and call_ds.startswith('x.__call__(...) <==> x(...)'):
                     call_ds = None
                 if call_ds:
                     out['call_docstring'] = call_ds
 
         # Compute the object's argspec as a callable.  The key is to decide
         # whether to pull it from the object itself, from its __init__ or
         # from its __call__ method.
 
         if inspect.isclass(obj):
             # Old-style classes need not have an __init__
             callable_obj = getattr(obj, "__init__", None)
         elif callable(obj):
             callable_obj = obj
         else:
             callable_obj = None
 
         if callable_obj:
             try:
                 args,  varargs, varkw, defaults = getargspec(callable_obj)
             except (TypeError, AttributeError):
                 # For extensions/builtins we can't retrieve the argspec
                 pass
             else:
                 out['argspec'] = dict(args=args, varargs=varargs,
                                       varkw=varkw, defaults=defaults)
 
         return object_info(**out)
 
 
     def psearch(self,pattern,ns_table,ns_search=[],
                 ignore_case=False,show_all=False):
         """Search namespaces with wildcards for objects.
 
         Arguments:
 
         - pattern: string containing shell-like wildcards to use in namespace
         searches and optionally a type specification to narrow the search to
         objects of that type.
 
         - ns_table: dict of name->namespaces for search.
 
         Optional arguments:
 
           - ns_search: list of namespace names to include in search.
 
           - ignore_case(False): make the search case-insensitive.
 
           - show_all(False): show all names, including those starting with
           underscores.
         """
         #print 'ps pattern:<%r>' % pattern # dbg
 
         # defaults
         type_pattern = 'all'
         filter = ''
 
         cmds = pattern.split()
         len_cmds  =  len(cmds)
         if len_cmds == 1:
             # Only filter pattern given
             filter = cmds[0]
         elif len_cmds == 2:
             # Both filter and type specified
             filter,type_pattern = cmds
         else:
             raise ValueError('invalid argument string for psearch: <%s>' %
                              pattern)
 
         # filter search namespaces
         for name in ns_search:
             if name not in ns_table:
                 raise ValueError('invalid namespace <%s>. Valid names: %s' %
                                  (name,ns_table.keys()))
 
         #print 'type_pattern:',type_pattern # dbg
         search_result, namespaces_seen = set(), set()
         for ns_name in ns_search:
             ns = ns_table[ns_name]
             # Normally, locals and globals are the same, so we just check one.
             if id(ns) in namespaces_seen:
                 continue
             namespaces_seen.add(id(ns))
             tmp_res = list_namespace(ns, type_pattern, filter,
                                     ignore_case=ignore_case, show_all=show_all)
             search_result.update(tmp_res)
 
         page.page('\n'.join(sorted(search_result)))

IPython/utils/io.py

@@ -1,409 +1,385 b''
 # encoding: utf-8
 """
 IO related utilities.
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 from __future__ import print_function
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 import re
 import os
 import sys
 import tempfile
 from StringIO import StringIO
 
 #-----------------------------------------------------------------------------
 # Code
 #-----------------------------------------------------------------------------
 
 
 class IOStream:
 
     def __init__(self,stream, fallback=None):
         if not hasattr(stream,'write') or not hasattr(stream,'flush'):
             if fallback is not None:
                 stream = fallback
             else:
                 raise ValueError("fallback required, but not specified")
         self.stream = stream
         self._swrite = stream.write
 
         # clone all methods not overridden:
         def clone(meth):
             return not hasattr(self, meth) and not meth.startswith('_')
         for meth in filter(clone, dir(stream)):
             setattr(self, meth, getattr(stream, meth))
 
     def write(self,data):
         try:
             self._swrite(data)
         except:
             try:
                 # print handles some unicode issues which may trip a plain
                 # write() call.  Emulate write() by using an empty end
                 # argument.
                 print(data, end='', file=self.stream)
             except:
                 # if we get here, something is seriously broken.
                 print('ERROR - failed to write data to stream:', self.stream,
                       file=sys.stderr)
 
     def writelines(self, lines):
         if isinstance(lines, basestring):
             lines = [lines]
         for line in lines:
             self.write(line)
 
     # This class used to have a writeln method, but regular files and streams
     # in Python don't have this method. We need to keep this completely
     # compatible so we removed it.
 
     @property
     def closed(self):
         return self.stream.closed
 
     def close(self):
         pass
 
 # setup stdin/stdout/stderr to sys.stdin/sys.stdout/sys.stderr
 devnull = open(os.devnull, 'a')
 stdin = IOStream(sys.stdin, fallback=devnull)
 stdout = IOStream(sys.stdout, fallback=devnull)
 stderr = IOStream(sys.stderr, fallback=devnull)
 
 class IOTerm:
     """ Term holds the file or file-like objects for handling I/O operations.
 
     These are normally just sys.stdin, sys.stdout and sys.stderr but for
     Windows they can can replaced to allow editing the strings before they are
     displayed."""
 
     # In the future, having IPython channel all its I/O operations through
     # this class will make it easier to embed it into other environments which
     # are not a normal terminal (such as a GUI-based shell)
     def __init__(self, stdin=None, stdout=None, stderr=None):
         mymodule = sys.modules[__name__]
         self.stdin  = IOStream(stdin, mymodule.stdin)
         self.stdout = IOStream(stdout, mymodule.stdout)
         self.stderr = IOStream(stderr, mymodule.stderr)
 
 
 class Tee(object):
     """A class to duplicate an output stream to stdout/err.
 
     This works in a manner very similar to the Unix 'tee' command.
 
     When the object is closed or deleted, it closes the original file given to
     it for duplication.
     """
     # Inspired by:
     # http://mail.python.org/pipermail/python-list/2007-May/442737.html
 
     def __init__(self, file_or_name, mode="w", channel='stdout'):
         """Construct a new Tee object.
 
         Parameters
         ----------
         file_or_name : filename or open filehandle (writable)
           File that will be duplicated
 
         mode : optional, valid mode for open().
           If a filename was give, open with this mode.
 
         channel : str, one of ['stdout', 'stderr']
         """
         if channel not in ['stdout', 'stderr']:
             raise ValueError('Invalid channel spec %s' % channel)
 
         if hasattr(file_or_name, 'write') and hasattr(file_or_name, 'seek'):
             self.file = file_or_name
         else:
             self.file = open(file_or_name, mode)
         self.channel = channel
         self.ostream = getattr(sys, channel)
         setattr(sys, channel, self)
         self._closed = False
 
     def close(self):
         """Close the file and restore the channel."""
         self.flush()
         setattr(sys, self.channel, self.ostream)
         self.file.close()
         self._closed = True
 
     def write(self, data):
         """Write data to both channels."""
         self.file.write(data)
         self.ostream.write(data)
         self.ostream.flush()
 
     def flush(self):
         """Flush both channels."""
         self.file.flush()
         self.ostream.flush()
 
     def __del__(self):
         if not self._closed:
             self.close()
 
 
-def guess_encoding(lines):
-    """check list of lines for line matching the source code encoding pattern
-    
-    Only check first two lines
-    """
-    reg = re.compile("#.*coding[:=]\s*([-\w.]+)")
-    for row in lines[:2]: #We only need to check the first two lines
-        result = reg.match(row)
-        if result:
-            coding = result.groups()[0]
-            break
-    else:
-        coding = "ascii"
-    return coding
-
-def source_to_unicode(txt):
-    """Converts string with python source code to unicode
-    """
-    if isinstance(txt, unicode):
-        return txt
-    coding = guess_encoding(txt.split("\n", 2))
-    return txt.decode(coding, errors="replace")
-    
-
 def file_read(filename):
     """Read a file and close it.  Returns the file source."""
     fobj = open(filename,'r');
     source = fobj.read();
     fobj.close()
     return source
 
 
 def file_readlines(filename):
     """Read a file and close it.  Returns the file source using readlines()."""
     fobj = open(filename,'r');
     lines = fobj.readlines();
     fobj.close()
     return lines
 
 
 def raw_input_multi(header='', ps1='==> ', ps2='..> ',terminate_str = '.'):
     """Take multiple lines of input.
 
     A list with each line of input as a separate element is returned when a
     termination string is entered (defaults to a single '.'). Input can also
     terminate via EOF (^D in Unix, ^Z-RET in Windows).
 
     Lines of input which end in \\ are joined into single entries (and a
     secondary continuation prompt is issued as long as the user terminates
     lines with \\). This allows entering very long strings which are still
     meant to be treated as single entities.
     """
 
     try:
         if header:
             header += '\n'
         lines = [raw_input(header + ps1)]
     except EOFError:
         return []
     terminate = [terminate_str]
     try:
         while lines[-1:] != terminate:
             new_line = raw_input(ps1)
             while new_line.endswith('\\'):
                 new_line = new_line[:-1] + raw_input(ps2)
             lines.append(new_line)
 
         return lines[:-1]  # don't return the termination command
     except EOFError:
         print()
         return lines
 
 
 def raw_input_ext(prompt='',  ps2='... '):
     """Similar to raw_input(), but accepts extended lines if input ends with \\."""
 
     line = raw_input(prompt)
     while line.endswith('\\'):
         line = line[:-1] + raw_input(ps2)
     return line
 
 
 def ask_yes_no(prompt,default=None):
     """Asks a question and returns a boolean (y/n) answer.
 
     If default is given (one of 'y','n'), it is used if the user input is
     empty. Otherwise the question is repeated until an answer is given.
 
     An EOF is treated as the default answer.  If there is no default, an
     exception is raised to prevent infinite loops.
 
     Valid answers are: y/yes/n/no (match is not case sensitive)."""
 
     answers = {'y':True,'n':False,'yes':True,'no':False}
     ans = None
     while ans not in answers.keys():
         try:
             ans = raw_input(prompt+' ').lower()
             if not ans:  # response was an empty string
                 ans = default
         except KeyboardInterrupt:
             pass
         except EOFError:
             if default in answers.keys():
                 ans = default
                 print()
             else:
                 raise
 
     return answers[ans]
 
 
 class NLprinter:
     """Print an arbitrarily nested list, indicating index numbers.
 
     An instance of this class called nlprint is available and callable as a
     function.
 
     nlprint(list,indent=' ',sep=': ') -> prints indenting each level by 'indent'
     and using 'sep' to separate the index from the value. """
 
     def __init__(self):
         self.depth = 0
 
     def __call__(self,lst,pos='',**kw):
         """Prints the nested list numbering levels."""
         kw.setdefault('indent',' ')
         kw.setdefault('sep',': ')
         kw.setdefault('start',0)
         kw.setdefault('stop',len(lst))
         # we need to remove start and stop from kw so they don't propagate
         # into a recursive call for a nested list.
         start = kw['start']; del kw['start']
         stop = kw['stop']; del kw['stop']
         if self.depth == 0 and 'header' in kw.keys():
             print(kw['header'])
 
         for idx in range(start,stop):
             elem = lst[idx]
             newpos = pos + str(idx)
             if type(elem)==type([]):
                 self.depth += 1
                 self.__call__(elem, newpos+",", **kw)
                 self.depth -= 1
             else:
                 print(kw['indent']*self.depth + newpos + kw["sep"] + repr(elem))
 
 nlprint = NLprinter()
 
 
 def temp_pyfile(src, ext='.py'):
     """Make a temporary python file, return filename and filehandle.
 
     Parameters
     ----------
     src : string or list of strings (no need for ending newlines if list)
       Source code to be written to the file.
 
     ext : optional, string
       Extension for the generated file.
 
     Returns
     -------
     (filename, open filehandle)
       It is the caller's responsibility to close the open file and unlink it.
     """
     fname = tempfile.mkstemp(ext)[1]
     f = open(fname,'w')
     f.write(src)
     f.flush()
     return fname, f
 
 
 def raw_print(*args, **kw):
     """Raw print to sys.__stdout__, otherwise identical interface to print()."""
 
     print(*args, sep=kw.get('sep', ' '), end=kw.get('end', '\n'),
           file=sys.__stdout__)
     sys.__stdout__.flush()
 
 
 def raw_print_err(*args, **kw):
     """Raw print to sys.__stderr__, otherwise identical interface to print()."""
 
     print(*args, sep=kw.get('sep', ' '), end=kw.get('end', '\n'),
           file=sys.__stderr__)
     sys.__stderr__.flush()
 
 
 # Short aliases for quick debugging, do NOT use these in production code.
 rprint = raw_print
 rprinte = raw_print_err
 
 
 class CapturedIO(object):
     """Simple object for containing captured stdout/err StringIO objects"""
     
     def __init__(self, stdout, stderr):
         self._stdout = stdout
         self._stderr = stderr
     
     def __str__(self):
         return self.stdout
     
     @property
     def stdout(self):
         if not self._stdout:
             return ''
         return self._stdout.getvalue()
     
     @property
     def stderr(self):
         if not self._stderr:
             return ''
         return self._stderr.getvalue()
     
     def show(self):
         """write my output to sys.stdout/err as appropriate"""
         sys.stdout.write(self.stdout)
         sys.stderr.write(self.stderr)
         sys.stdout.flush()
         sys.stderr.flush()
     
     __call__ = show
 
 
 class capture_output(object):
     """context manager for capturing stdout/err"""
     stdout = True
     stderr = True
     
     def __init__(self, stdout=True, stderr=True):
         self.stdout = stdout
         self.stderr = stderr
     
     def __enter__(self):
         self.sys_stdout = sys.stdout
         self.sys_stderr = sys.stderr
         
         stdout = stderr = False
         if self.stdout:
             stdout = sys.stdout = StringIO()
         if self.stderr:
             stderr = sys.stderr = StringIO()
         
         return CapturedIO(stdout, stderr)
     
     def __exit__(self, exc_type, exc_value, traceback):
         sys.stdout = self.sys_stdout
         sys.stderr = self.sys_stderr
 
 

IPython/utils/openpy.py

@@ -1,192 +1,204 b''
 """
 Tools to open .py files as Unicode, using the encoding specified within the file,
 as per PEP 263.
 
 Much of the code is taken from the tokenize module in Python 3.2.
 """
 from __future__ import absolute_import
 
 import io
 from io import TextIOWrapper
 import re
+from StringIO import StringIO
 import urllib
 
 cookie_re = re.compile(ur"coding[:=]\s*([-\w.]+)", re.UNICODE)
 cookie_comment_re = re.compile(ur"^\s*#.*coding[:=]\s*([-\w.]+)", re.UNICODE)
 
 try:
     # Available in Python 3
     from tokenize import detect_encoding
 except ImportError:
     from codecs import lookup, BOM_UTF8
     
     # Copied from Python 3.2 tokenize
     def _get_normal_name(orig_enc):
         """Imitates get_normal_name in tokenizer.c."""
         # Only care about the first 12 characters.
         enc = orig_enc[:12].lower().replace("_", "-")
         if enc == "utf-8" or enc.startswith("utf-8-"):
             return "utf-8"
         if enc in ("latin-1", "iso-8859-1", "iso-latin-1") or \
            enc.startswith(("latin-1-", "iso-8859-1-", "iso-latin-1-")):
             return "iso-8859-1"
         return orig_enc
     
     # Copied from Python 3.2 tokenize
     def detect_encoding(readline):
         """
         The detect_encoding() function is used to detect the encoding that should
         be used to decode a Python source file.  It requires one argment, readline,
         in the same way as the tokenize() generator.
 
         It will call readline a maximum of twice, and return the encoding used
         (as a string) and a list of any lines (left as bytes) it has read in.
 
         It detects the encoding from the presence of a utf-8 bom or an encoding
         cookie as specified in pep-0263.  If both a bom and a cookie are present,
         but disagree, a SyntaxError will be raised.  If the encoding cookie is an
         invalid charset, raise a SyntaxError.  Note that if a utf-8 bom is found,
         'utf-8-sig' is returned.
 
         If no encoding is specified, then the default of 'utf-8' will be returned.
         """
         bom_found = False
         encoding = None
         default = 'utf-8'
         def read_or_stop():
             try:
                 return readline()
             except StopIteration:
                 return b''
 
         def find_cookie(line):
             try:
                 line_string = line.decode('ascii')
             except UnicodeDecodeError:
                 return None
 
             matches = cookie_re.findall(line_string)
             if not matches:
                 return None
             encoding = _get_normal_name(matches[0])
             try:
                 codec = lookup(encoding)
             except LookupError:
                 # This behaviour mimics the Python interpreter
                 raise SyntaxError("unknown encoding: " + encoding)
 
             if bom_found:
                 if codec.name != 'utf-8':
                     # This behaviour mimics the Python interpreter
                     raise SyntaxError('encoding problem: utf-8')
                 encoding += '-sig'
             return encoding
 
         first = read_or_stop()
         if first.startswith(BOM_UTF8):
             bom_found = True
             first = first[3:]
             default = 'utf-8-sig'
         if not first:
             return default, []
 
         encoding = find_cookie(first)
         if encoding:
             return encoding, [first]
 
         second = read_or_stop()
         if not second:
             return default, [first]
 
         encoding = find_cookie(second)
         if encoding:
             return encoding, [first, second]
 
         return default, [first, second]
 
 try:
     # Available in Python 3.2 and above.
     from tokenize import open
 except ImportError:
     # Copied from Python 3.2 tokenize
     def open(filename):
         """Open a file in read only mode using the encoding detected by
         detect_encoding().
         """
         buffer = io.open(filename, 'rb')   # Tweaked to use io.open for Python 2
         encoding, lines = detect_encoding(buffer.readline)
         buffer.seek(0)
         text = TextIOWrapper(buffer, encoding, line_buffering=True)
         text.mode = 'r'
         return text   
 
+def source_to_unicode(txt):
+    """Converts string with python source code to unicode
+    """
+    if isinstance(txt, unicode):
+        return txt
+    try:
+        coding, _ = detect_encoding(StringIO(txt).readline)
+    except SyntaxError:
+        coding = "ascii"
+    return txt.decode(coding, errors="replace")
+
 def strip_encoding_cookie(filelike):
     """Generator to pull lines from a text-mode file, skipping the encoding
     cookie if it is found in the first two lines.
     """
     it = iter(filelike)
     try:
         first = next(it)
         if not cookie_comment_re.match(first):
             yield first
         second = next(it)
         if not cookie_comment_re.match(second):
             yield second
     except StopIteration:
         return
     
     for line in it:
         yield line
 
 def read_py_file(filename, skip_encoding_cookie=True):
     """Read a Python file, using the encoding declared inside the file.
     
     Parameters
     ----------
     filename : str
       The path to the file to read.
     skip_encoding_cookie : bool
       If True (the default), and the encoding declaration is found in the first
       two lines, that line will be excluded from the output - compiling a
       unicode string with an encoding declaration is a SyntaxError in Python 2.
     
     Returns
     -------
     A unicode string containing the contents of the file.
     """
     with open(filename) as f:   # the open function defined in this module.
         if skip_encoding_cookie:
             return "".join(strip_encoding_cookie(f))
         else:
             return f.read()
 
 def read_py_url(url, errors='replace', skip_encoding_cookie=True):
     """Read a Python file from a URL, using the encoding declared inside the file.
     
     Parameters
     ----------
     url : str
       The URL from which to fetch the file.
     errors : str
       How to handle decoding errors in the file. Options are the same as for
       bytes.decode(), but here 'replace' is the default.
     skip_encoding_cookie : bool
       If True (the default), and the encoding declaration is found in the first
       two lines, that line will be excluded from the output - compiling a
       unicode string with an encoding declaration is a SyntaxError in Python 2.
     
     Returns
     -------
     A unicode string containing the contents of the file.
     """
     response = urllib.urlopen(url)
     buffer = io.BytesIO(response.read())
     encoding, lines = detect_encoding(buffer.readline)
     buffer.seek(0)
     text = TextIOWrapper(buffer, encoding, errors=errors, line_buffering=True)
     text.mode = 'r'
     if skip_encoding_cookie:
         return "".join(strip_encoding_cookie(text))
     else:
         return text.read()
 

IPython/zmq/zmqshell.py

@@ -1,588 +1,590 b''
 """A ZMQ-based subclass of InteractiveShell.
 
 This code is meant to ease the refactoring of the base InteractiveShell into
 something with a cleaner architecture for 2-process use, without actually
 breaking InteractiveShell itself.  So we're doing something a bit ugly, where
 we subclass and override what we want to fix.  Once this is working well, we
 can go back to the base class and refactor the code for a cleaner inheritance
 implementation that doesn't rely on so much monkeypatching.
 
 But this lets us maintain a fully working IPython as we develop the new
 machinery.  This should thus be thought of as scaffolding.
 """
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 from __future__ import print_function
 
 # Stdlib
 import os
 import sys
 import time
 
 # System library imports
 from zmq.eventloop import ioloop
 
 # Our own
 from IPython.core.interactiveshell import (
     InteractiveShell, InteractiveShellABC
 )
 from IPython.core import page
 from IPython.core.autocall import ZMQExitAutocall
 from IPython.core.displaypub import DisplayPublisher
 from IPython.core.error import UsageError
 from IPython.core.magics import MacroToEdit, CodeMagics
 from IPython.core.magic import magics_class, line_magic, Magics
 from IPython.core.payloadpage import install_payload_page
 from IPython.lib.kernel import (
     get_connection_file, get_connection_info, connect_qtconsole
 )
 from IPython.testing.skipdoctest import skip_doctest
-from IPython.utils import io
+from IPython.utils import io, openpy
 from IPython.utils.jsonutil import json_clean, encode_images
 from IPython.utils.process import arg_split
 from IPython.utils import py3compat
 from IPython.utils.traitlets import Instance, Type, Dict, CBool, CBytes
 from IPython.utils.warn import warn, error
 from IPython.zmq.displayhook import ZMQShellDisplayHook
 from IPython.zmq.datapub import ZMQDataPublisher
 from IPython.zmq.session import extract_header
 from session import Session
 
 #-----------------------------------------------------------------------------
 # Functions and classes
 #-----------------------------------------------------------------------------
 
 class ZMQDisplayPublisher(DisplayPublisher):
     """A display publisher that publishes data using a ZeroMQ PUB socket."""
 
     session = Instance(Session)
     pub_socket = Instance('zmq.Socket')
     parent_header = Dict({})
     topic = CBytes(b'displaypub')
 
     def set_parent(self, parent):
         """Set the parent for outbound messages."""
         self.parent_header = extract_header(parent)
     
     def _flush_streams(self):
         """flush IO Streams prior to display"""
         sys.stdout.flush()
         sys.stderr.flush()
 
     def publish(self, source, data, metadata=None):
         self._flush_streams()
         if metadata is None:
             metadata = {}
         self._validate_data(source, data, metadata)
         content = {}
         content['source'] = source
         content['data'] = encode_images(data)
         content['metadata'] = metadata
         self.session.send(
             self.pub_socket, u'display_data', json_clean(content),
             parent=self.parent_header, ident=self.topic,
         )
 
     def clear_output(self, stdout=True, stderr=True, other=True):
         content = dict(stdout=stdout, stderr=stderr, other=other)
         
         if stdout:
             print('\r', file=sys.stdout, end='')
         if stderr:
             print('\r', file=sys.stderr, end='')
         
         self._flush_streams()
         
         self.session.send(
             self.pub_socket, u'clear_output', content,
             parent=self.parent_header, ident=self.topic,
         )
 
 @magics_class
 class KernelMagics(Magics):
     #------------------------------------------------------------------------
     # Magic overrides
     #------------------------------------------------------------------------
     # Once the base class stops inheriting from magic, this code needs to be
     # moved into a separate machinery as well.  For now, at least isolate here
     # the magics which this class needs to implement differently from the base
     # class, or that are unique to it.
 
     @line_magic
     def doctest_mode(self, parameter_s=''):
         """Toggle doctest mode on and off.
 
         This mode is intended to make IPython behave as much as possible like a
         plain Python shell, from the perspective of how its prompts, exceptions
         and output look.  This makes it easy to copy and paste parts of a
         session into doctests.  It does so by:
 
         - Changing the prompts to the classic ``>>>`` ones.
         - Changing the exception reporting mode to 'Plain'.
         - Disabling pretty-printing of output.
 
         Note that IPython also supports the pasting of code snippets that have
         leading '>>>' and '...' prompts in them.  This means that you can paste
         doctests from files or docstrings (even if they have leading
         whitespace), and the code will execute correctly.  You can then use
         '%history -t' to see the translated history; this will give you the
         input after removal of all the leading prompts and whitespace, which
         can be pasted back into an editor.
 
         With these features, you can switch into this mode easily whenever you
         need to do testing and changes to doctests, without having to leave
         your existing IPython session.
         """
 
         from IPython.utils.ipstruct import Struct
 
         # Shorthands
         shell = self.shell
         disp_formatter = self.shell.display_formatter
         ptformatter = disp_formatter.formatters['text/plain']
         # dstore is a data store kept in the instance metadata bag to track any
         # changes we make, so we can undo them later.
         dstore = shell.meta.setdefault('doctest_mode', Struct())
         save_dstore = dstore.setdefault
 
         # save a few values we'll need to recover later
         mode = save_dstore('mode', False)
         save_dstore('rc_pprint', ptformatter.pprint)
         save_dstore('rc_plain_text_only',disp_formatter.plain_text_only)
         save_dstore('xmode', shell.InteractiveTB.mode)
 
         if mode == False:
             # turn on
             ptformatter.pprint = False
             disp_formatter.plain_text_only = True
             shell.magic('xmode Plain')
         else:
             # turn off
             ptformatter.pprint = dstore.rc_pprint
             disp_formatter.plain_text_only = dstore.rc_plain_text_only
             shell.magic("xmode " + dstore.xmode)
 
         # Store new mode and inform on console
         dstore.mode = bool(1-int(mode))
         mode_label = ['OFF','ON'][dstore.mode]
         print('Doctest mode is:', mode_label)
 
         # Send the payload back so that clients can modify their prompt display
         payload = dict(
             source='IPython.zmq.zmqshell.ZMQInteractiveShell.doctest_mode',
             mode=dstore.mode)
         shell.payload_manager.write_payload(payload)
         
     
     _find_edit_target = CodeMagics._find_edit_target
 
     @skip_doctest
     @line_magic
     def edit(self, parameter_s='', last_call=['','']):
         """Bring up an editor and execute the resulting code.
 
         Usage:
           %edit [options] [args]
 
         %edit runs an external text editor. You will need to set the command for
         this editor via the ``TerminalInteractiveShell.editor`` option in your
         configuration file before it will work.
 
         This command allows you to conveniently edit multi-line code right in
         your IPython session.
 
         If called without arguments, %edit opens up an empty editor with a
         temporary file and will execute the contents of this file when you
         close it (don't forget to save it!).
 
 
         Options:
 
         -n <number>: open the editor at a specified line number.  By default,
         the IPython editor hook uses the unix syntax 'editor +N filename', but
         you can configure this by providing your own modified hook if your
         favorite editor supports line-number specifications with a different
         syntax.
 
         -p: this will call the editor with the same data as the previous time
         it was used, regardless of how long ago (in your current session) it
         was.
 
         -r: use 'raw' input.  This option only applies to input taken from the
         user's history.  By default, the 'processed' history is used, so that
         magics are loaded in their transformed version to valid Python.  If
         this option is given, the raw input as typed as the command line is
         used instead.  When you exit the editor, it will be executed by
         IPython's own processor.
 
         -x: do not execute the edited code immediately upon exit. This is
         mainly useful if you are editing programs which need to be called with
         command line arguments, which you can then do using %run.
 
 
         Arguments:
 
         If arguments are given, the following possibilites exist:
 
         - The arguments are numbers or pairs of colon-separated numbers (like
         1 4:8 9). These are interpreted as lines of previous input to be
         loaded into the editor. The syntax is the same of the %macro command.
 
         - If the argument doesn't start with a number, it is evaluated as a
         variable and its contents loaded into the editor. You can thus edit
         any string which contains python code (including the result of
         previous edits).
 
         - If the argument is the name of an object (other than a string),
         IPython will try to locate the file where it was defined and open the
         editor at the point where it is defined. You can use `%edit function`
         to load an editor exactly at the point where 'function' is defined,
         edit it and have the file be executed automatically.
 
         If the object is a macro (see %macro for details), this opens up your
         specified editor with a temporary file containing the macro's data.
         Upon exit, the macro is reloaded with the contents of the file.
 
         Note: opening at an exact line is only supported under Unix, and some
         editors (like kedit and gedit up to Gnome 2.8) do not understand the
         '+NUMBER' parameter necessary for this feature. Good editors like
         (X)Emacs, vi, jed, pico and joe all do.
 
         - If the argument is not found as a variable, IPython will look for a
         file with that name (adding .py if necessary) and load it into the
         editor. It will execute its contents with execfile() when you exit,
         loading any code in the file into your interactive namespace.
 
         After executing your code, %edit will return as output the code you
         typed in the editor (except when it was an existing file). This way
         you can reload the code in further invocations of %edit as a variable,
         via _<NUMBER> or Out[<NUMBER>], where <NUMBER> is the prompt number of
         the output.
 
         Note that %edit is also available through the alias %ed.
 
         This is an example of creating a simple function inside the editor and
         then modifying it. First, start up the editor:
 
         In [1]: ed
         Editing... done. Executing edited code...
         Out[1]: 'def foo():n    print "foo() was defined in an editing session"n'
 
         We can then call the function foo():
 
         In [2]: foo()
         foo() was defined in an editing session
 
         Now we edit foo.  IPython automatically loads the editor with the
         (temporary) file where foo() was previously defined:
 
         In [3]: ed foo
         Editing... done. Executing edited code...
 
         And if we call foo() again we get the modified version:
 
         In [4]: foo()
         foo() has now been changed!
 
         Here is an example of how to edit a code snippet successive
         times. First we call the editor:
 
         In [5]: ed
         Editing... done. Executing edited code...
         hello
         Out[5]: "print 'hello'n"
 
         Now we call it again with the previous output (stored in _):
 
         In [6]: ed _
         Editing... done. Executing edited code...
         hello world
         Out[6]: "print 'hello world'n"
 
         Now we call it with the output #8 (stored in _8, also as Out[8]):
 
         In [7]: ed _8
         Editing... done. Executing edited code...
         hello again
         Out[7]: "print 'hello again'n"
         """
 
         opts,args = self.parse_options(parameter_s,'prn:')
 
         try:
             filename, lineno, _ = CodeMagics._find_edit_target(self.shell, args, opts, last_call)
         except MacroToEdit as e:
             # TODO: Implement macro editing over 2 processes.
             print("Macro editing not yet implemented in 2-process model.")
             return
 
         # Make sure we send to the client an absolute path, in case the working
         # directory of client and kernel don't match
         filename = os.path.abspath(filename)
 
         payload = {
             'source' : 'IPython.zmq.zmqshell.ZMQInteractiveShell.edit_magic',
             'filename' : filename,
             'line_number' : lineno
         }
         self.shell.payload_manager.write_payload(payload)
 
     # A few magics that are adapted to the specifics of using pexpect and a
     # remote terminal
 
     @line_magic
     def clear(self, arg_s):
         """Clear the terminal."""
         if os.name == 'posix':
             self.shell.system("clear")
         else:
             self.shell.system("cls")
 
     if os.name == 'nt':
         # This is the usual name in windows
         cls = line_magic('cls')(clear)
 
     # Terminal pagers won't work over pexpect, but we do have our own pager
 
     @line_magic
     def less(self, arg_s):
         """Show a file through the pager.
 
         Files ending in .py are syntax-highlighted."""
         if not arg_s:
             raise UsageError('Missing filename.')
 
         cont = open(arg_s).read()
         if arg_s.endswith('.py'):
-            cont = self.shell.pycolorize(io.source_to_unicode(cont))
+            cont = self.shell.pycolorize(openpy.read_py_file(arg_s, skip_encoding_cookie=False))
+        else:
+            cont = open(arg_s).read()
         page.page(cont)
 
     more = line_magic('more')(less)
 
     # Man calls a pager, so we also need to redefine it
     if os.name == 'posix':
         @line_magic
         def man(self, arg_s):
             """Find the man page for the given command and display in pager."""
             page.page(self.shell.getoutput('man %s | col -b' % arg_s,
                                            split=False))
 
     @line_magic
     def connect_info(self, arg_s):
         """Print information for connecting other clients to this kernel
         
         It will print the contents of this session's connection file, as well as
         shortcuts for local clients.
         
         In the simplest case, when called from the most recently launched kernel,
         secondary clients can be connected, simply with:
         
         $> ipython <app> --existing
         
         """
         
         from IPython.core.application import BaseIPythonApplication as BaseIPApp
         
         if BaseIPApp.initialized():
             app = BaseIPApp.instance()
             security_dir = app.profile_dir.security_dir
             profile = app.profile
         else:
             profile = 'default'
             security_dir = ''
         
         try:
             connection_file = get_connection_file()
             info = get_connection_info(unpack=False)
         except Exception as e:
             error("Could not get connection info: %r" % e)
             return
         
         # add profile flag for non-default profile
         profile_flag = "--profile %s" % profile if profile != 'default' else ""
         
         # if it's in the security dir, truncate to basename
         if security_dir == os.path.dirname(connection_file):
             connection_file = os.path.basename(connection_file)
         
         
         print (info + '\n')
         print ("Paste the above JSON into a file, and connect with:\n"
             "    $> ipython <app> --existing <file>\n"
             "or, if you are local, you can connect with just:\n"
             "    $> ipython <app> --existing {0} {1}\n"
             "or even just:\n"
             "    $> ipython <app> --existing {1}\n"
             "if this is the most recent IPython session you have started.".format(
             connection_file, profile_flag
             )
         )
 
     @line_magic
     def qtconsole(self, arg_s):
         """Open a qtconsole connected to this kernel.
         
         Useful for connecting a qtconsole to running notebooks, for better
         debugging.
         """
         
         # %qtconsole should imply bind_kernel for engines:
         try:
             from IPython.parallel import bind_kernel
         except ImportError:
             # technically possible, because parallel has higher pyzmq min-version
             pass
         else:
             bind_kernel()
         
         try:
             p = connect_qtconsole(argv=arg_split(arg_s, os.name=='posix'))
         except Exception as e:
             error("Could not start qtconsole: %r" % e)
             return
 
 def safe_unicode(e):
     """unicode(e) with various fallbacks. Used for exceptions, which may not be
     safe to call unicode() on.
     """
     try:
         return unicode(e)
     except UnicodeError:
         pass
 
     try:
         return py3compat.str_to_unicode(str(e))
     except UnicodeError:
         pass
 
     try:
         return py3compat.str_to_unicode(repr(e))
     except UnicodeError:
         pass
 
     return u'Unrecoverably corrupt evalue'
 
 
 class ZMQInteractiveShell(InteractiveShell):
     """A subclass of InteractiveShell for ZMQ."""
 
     displayhook_class = Type(ZMQShellDisplayHook)
     display_pub_class = Type(ZMQDisplayPublisher)
     data_pub_class = Type(ZMQDataPublisher)
 
     # Override the traitlet in the parent class, because there's no point using
     # readline for the kernel. Can be removed when the readline code is moved
     # to the terminal frontend.
     colors_force = CBool(True)
     readline_use = CBool(False)
     # autoindent has no meaning in a zmqshell, and attempting to enable it
     # will print a warning in the absence of readline.
     autoindent = CBool(False)
 
     exiter = Instance(ZMQExitAutocall)
     def _exiter_default(self):
         return ZMQExitAutocall(self)
     
     def _exit_now_changed(self, name, old, new):
         """stop eventloop when exit_now fires"""
         if new:
             loop = ioloop.IOLoop.instance()
             loop.add_timeout(time.time()+0.1, loop.stop)
 
     keepkernel_on_exit = None
 
     # Over ZeroMQ, GUI control isn't done with PyOS_InputHook as there is no
     # interactive input being read; we provide event loop support in ipkernel
     from .eventloops import enable_gui
     enable_gui = staticmethod(enable_gui)
 
     def init_environment(self):
         """Configure the user's environment.
 
         """
         env = os.environ
         # These two ensure 'ls' produces nice coloring on BSD-derived systems
         env['TERM'] = 'xterm-color'
         env['CLICOLOR'] = '1'
         # Since normal pagers don't work at all (over pexpect we don't have
         # single-key control of the subprocess), try to disable paging in
         # subprocesses as much as possible.
         env['PAGER'] = 'cat'
         env['GIT_PAGER'] = 'cat'
         
         # And install the payload version of page.
         install_payload_page()
 
     def auto_rewrite_input(self, cmd):
         """Called to show the auto-rewritten input for autocall and friends.
 
         FIXME: this payload is currently not correctly processed by the
         frontend.
         """
         new = self.prompt_manager.render('rewrite') + cmd
         payload = dict(
             source='IPython.zmq.zmqshell.ZMQInteractiveShell.auto_rewrite_input',
             transformed_input=new,
             )
         self.payload_manager.write_payload(payload)
 
     def ask_exit(self):
         """Engage the exit actions."""
         self.exit_now = True
         payload = dict(
             source='IPython.zmq.zmqshell.ZMQInteractiveShell.ask_exit',
             exit=True,
             keepkernel=self.keepkernel_on_exit,
             )
         self.payload_manager.write_payload(payload)
 
     def _showtraceback(self, etype, evalue, stb):
 
         exc_content = {
             u'traceback' : stb,
             u'ename' : unicode(etype.__name__),
             u'evalue' : safe_unicode(evalue)
         }
 
         dh = self.displayhook
         # Send exception info over pub socket for other clients than the caller
         # to pick up
         topic = None
         if dh.topic:
             topic = dh.topic.replace(b'pyout', b'pyerr')
         
         exc_msg = dh.session.send(dh.pub_socket, u'pyerr', json_clean(exc_content), dh.parent_header, ident=topic)
 
         # FIXME - Hack: store exception info in shell object.  Right now, the
         # caller is reading this info after the fact, we need to fix this logic
         # to remove this hack.  Even uglier, we need to store the error status
         # here, because in the main loop, the logic that sets it is being
         # skipped because runlines swallows the exceptions.
         exc_content[u'status'] = u'error'
         self._reply_content = exc_content
         # /FIXME
 
         return exc_content
 
     def set_next_input(self, text):
         """Send the specified text to the frontend to be presented at the next
         input cell."""
         payload = dict(
             source='IPython.zmq.zmqshell.ZMQInteractiveShell.set_next_input',
             text=text
         )
         self.payload_manager.write_payload(payload)
     
     #-------------------------------------------------------------------------
     # Things related to magics
     #-------------------------------------------------------------------------
 
     def init_magics(self):
         super(ZMQInteractiveShell, self).init_magics()
         self.register_magics(KernelMagics)
         self.magics_manager.register_alias('ed', 'edit')
 
 
 
 InteractiveShellABC.register(ZMQInteractiveShell)