Merge pull request #1715 from jstenar/ultratb-pycolorize-unicode

Fix for  #1688, traceback-unicode issue

IPython/utils/ulinecache.py

@@ -0,0 +1,45 b''
+"""Wrapper around linecache which decodes files to unicode according to PEP 263.
+
+This is only needed for Python 2 - linecache in Python 3 does the same thing
+itself.
+"""
+import functools
+import linecache
+import sys
+
+from IPython.utils import py3compat
+from IPython.utils import openpy
+
+if py3compat.PY3:
+    getline = linecache.getline
+    
+    # getlines has to be looked up at runtime, because doctests monkeypatch it.
+    @functools.wraps(linecache.getlines)
+    def getlines(filename, module_globals=None):
+        return linecache.getlines(filename, module_globals=module_globals)
+
+else:
+    def getlines(filename, module_globals=None):
+        """Get the lines (as unicode) for a file from the cache.
+        Update the cache if it doesn't contain an entry for this file already."""
+        filename = py3compat.cast_bytes(filename, sys.getfilesystemencoding())
+        lines = linecache.getlines(filename, module_globals=module_globals)
+        
+        # The bits we cache ourselves can be unicode.
+        if (not lines) or isinstance(lines[0], unicode):
+            return lines
+        
+        readline = openpy._list_readline(lines)
+        try:
+            encoding, _ = openpy.detect_encoding(readline)
+        except SyntaxError:
+            encoding = 'ascii'
+        return [l.decode(encoding, 'replace') for l in lines]
+
+    # This is a straight copy of linecache.getline
+    def getline(filename, lineno, module_globals=None):
+        lines = getlines(filename, module_globals)
+        if 1 <= lineno <= len(lines):
+            return lines[lineno-1]
+        else:
+            return ''

IPython/core/debugger.py

@@ -30,9 +30,9 b' import bdb'
 import linecache
 import sys
 
-from IPython.utils import PyColorize
+from IPython.utils import PyColorize, ulinecache
 from IPython.core import ipapi
-from IPython.utils import coloransi, io
+from IPython.utils import coloransi, io, openpy, py3compat
 from IPython.core.excolors import exception_colors
 
 # See if we can use pydb.
@@ -304,16 +304,16 b' class Pdb(OldPdb):'
         # vds: <<
 
     def format_stack_entry(self, frame_lineno, lprefix=': ', context = 3):
-        import linecache, repr
+        import repr
 
         ret = []
 
         Colors = self.color_scheme_table.active_colors
         ColorsNormal = Colors.Normal
-        tpl_link = '%s%%s%s' % (Colors.filenameEm, ColorsNormal)
-        tpl_call = '%s%%s%s%%s%s' % (Colors.vName, Colors.valEm, ColorsNormal)
-        tpl_line = '%%s%s%%s %s%%s' % (Colors.lineno, ColorsNormal)
-        tpl_line_em = '%%s%s%%s %s%%s%s' % (Colors.linenoEm, Colors.line,
+        tpl_link = u'%s%%s%s' % (Colors.filenameEm, ColorsNormal)
+        tpl_call = u'%s%%s%s%%s%s' % (Colors.vName, Colors.valEm, ColorsNormal)
+        tpl_line = u'%%s%s%%s %s%%s' % (Colors.lineno, ColorsNormal)
+        tpl_line_em = u'%%s%s%%s %s%%s%s' % (Colors.linenoEm, Colors.line,
                                             ColorsNormal)
 
         frame, lineno = frame_lineno
@@ -327,7 +327,7 b' class Pdb(OldPdb):'
 
         #s = filename + '(' + `lineno` + ')'
         filename = self.canonic(frame.f_code.co_filename)
-        link = tpl_link % filename
+        link = tpl_link % py3compat.cast_unicode(filename)
 
         if frame.f_code.co_name:
             func = frame.f_code.co_name
@@ -348,10 +348,10 b' class Pdb(OldPdb):'
             ret.append('> ')
         else:
             ret.append('  ')
-        ret.append('%s(%s)%s\n' % (link,lineno,call))
+        ret.append(u'%s(%s)%s\n' % (link,lineno,call))
 
         start = lineno - 1 - context//2
-        lines = linecache.getlines(filename)
+        lines = ulinecache.getlines(filename)
         start = max(start, 0)
         start = min(start, len(lines) - context)
         lines = lines[start : start + context]
@@ -364,7 +364,6 b' class Pdb(OldPdb):'
             ret.append(self.__format_line(linetpl, filename,
                                           start + 1 + i, line,
                                           arrow = show_arrow) )
-
         return ''.join(ret)
 
     def __format_line(self, tpl_line, filename, lineno, line, arrow = False):
@@ -422,8 +421,11 b' class Pdb(OldPdb):'
             tpl_line = '%%s%s%%s %s%%s' % (Colors.lineno, ColorsNormal)
             tpl_line_em = '%%s%s%%s %s%%s%s' % (Colors.linenoEm, Colors.line, ColorsNormal)
             src = []
+            if filename == "<string>" and hasattr(self, "_exec_filename"):
+                filename = self._exec_filename
+            
             for lineno in range(first, last+1):
-                line = linecache.getline(filename, lineno)
+                line = ulinecache.getline(filename, lineno)
                 if not line:
                     break
 

IPython/core/interactiveshell.py

@@ -2908,7 +2908,7 b' class InteractiveShell(SingletonConfigurable):'
         lines = self.history_manager.get_range_by_str(range_str, raw=raw)
         return "\n".join(x for _, _, x in lines)
 
-    def find_user_code(self, target, raw=True, py_only=False):
+    def find_user_code(self, target, raw=True, py_only=False, skip_encoding_cookie=True):
         """Get a code string from history, file, url, or a string or macro.
 
         This is mainly used by magic functions.
@@ -2945,7 +2945,7 b' class InteractiveShell(SingletonConfigurable):'
         utarget = unquote_filename(target)
         try:
             if utarget.startswith(('http://', 'https://')):
-                return openpy.read_py_url(utarget, skip_encoding_cookie=True)
+                return openpy.read_py_url(utarget, skip_encoding_cookie=skip_encoding_cookie)
         except UnicodeDecodeError:
             if not py_only :
                 response = urllib.urlopen(target)
@@ -2961,7 +2961,7 b' class InteractiveShell(SingletonConfigurable):'
         for tgt in potential_target :
             if os.path.isfile(tgt):                        # Read file
                 try :
-                    return openpy.read_py_file(tgt, skip_encoding_cookie=True)
+                    return openpy.read_py_file(tgt, skip_encoding_cookie=skip_encoding_cookie)
                 except UnicodeDecodeError :
                     if not py_only :
                         with io_open(tgt,'r', encoding='latin1') as f :

IPython/core/magics/execution.py

@@ -554,6 +554,8 b' python-profiler package from non-free.""")'
                         print "%s prompt to start your script." % deb.prompt
                         ns = {'execfile': py3compat.execfile, 'prog_ns': prog_ns}
                         try:
+                            #save filename so it can be used by methods on the deb object
+                            deb._exec_filename = filename
                             deb.run('execfile("%s", prog_ns)' % filename, ns)
 
                         except:

IPython/core/magics/namespace.py

@@ -23,6 +23,7 b' from IPython.core.error import StdinNotImplementedError, UsageError'
 from IPython.core.magic import Magics, magics_class, line_magic
 from IPython.testing.skipdoctest import skip_doctest
 from IPython.utils.encoding import DEFAULT_ENCODING
+from IPython.utils.openpy import read_py_file
 from IPython.utils.path import get_py_filename
 
 #-----------------------------------------------------------------------------
@@ -118,7 +119,7 b' class NamespaceMagics(Magics):'
             except IOError as msg:
                 print msg
                 return
-            page.page(self.shell.inspector.format(open(filename).read()))
+            page.page(self.shell.pycolorize(read_py_file(filename, skip_encoding_cookie=False)))
 
     @line_magic
     def psearch(self, parameter_s=''):

IPython/core/magics/osm.py

@@ -32,6 +32,7 b' from IPython.core.magic import ('
 )
 from IPython.testing.skipdoctest import skip_doctest
 from IPython.utils.io import file_read, nlprint
+from IPython.utils.openpy import source_to_unicode
 from IPython.utils.path import get_py_filename, unquote_filename
 from IPython.utils.process import abbrev_cwd
 from IPython.utils.terminal import set_term_title
@@ -686,12 +687,12 b' class OSMagics(Magics):'
                              'or macro.')
 
         try :
-            cont = self.shell.find_user_code(parameter_s)
+            cont = self.shell.find_user_code(parameter_s, skip_encoding_cookie=False)
         except (ValueError, IOError):
             print "Error: no such file, variable, URL, history range or macro"
             return
 
-        page.page(self.shell.pycolorize(cont))
+        page.page(self.shell.pycolorize(source_to_unicode(cont)))
 
     @magic_arguments.magic_arguments()
     @magic_arguments.argument(

IPython/core/oinspect.py

@@ -24,6 +24,8 b' import linecache'
 import os
 import sys
 import types
+import io as stdlib_io
+
 from collections import namedtuple
 try:
     from itertools import izip_longest
@@ -35,10 +37,12 b' from IPython.core import page'
 from IPython.testing.skipdoctest import skip_doctest_py3
 from IPython.utils import PyColorize
 from IPython.utils import io
+from IPython.utils import openpy
 from IPython.utils import py3compat
 from IPython.utils.text import indent
 from IPython.utils.wildcard import list_namespace
 from IPython.utils.coloransi import *
+from IPython.utils.py3compat import cast_unicode
 
 #****************************************************************************
 # Builtin color schemes
@@ -90,6 +94,29 b' def object_info(**kw):'
     return infodict
 
 
+def get_encoding(obj):
+    """Get encoding for python source file defining obj
+
+    Returns None if obj is not defined in a sourcefile.
+    """
+    ofile = find_file(obj)
+    # run contents of file through pager starting at line where the object
+    # is defined, as long as the file isn't binary and is actually on the
+    # filesystem.
+    if ofile is None:
+        return None
+    elif ofile.endswith(('.so', '.dll', '.pyd')):
+        return None
+    elif not os.path.isfile(ofile):
+        return None
+    else:
+        # Print only text files, not extension binaries.  Note that
+        # getsourcelines returns lineno with 1-offset and page() uses
+        # 0-offset, so we must adjust.
+        buffer = stdlib_io.open(ofile, 'rb')   # Tweaked to use io.open for Python 2
+        encoding, lines = openpy.detect_encoding(buffer.readline)
+        return encoding
+
 def getdoc(obj):
     """Stable wrapper around inspect.getdoc.
 
@@ -109,10 +136,13 b' def getdoc(obj):'
             return inspect.cleandoc(ds)
     
     try:
-        return inspect.getdoc(obj)
+        docstr = inspect.getdoc(obj)
+        encoding = get_encoding(obj)
+        return py3compat.cast_unicode(docstr, encoding=encoding)
     except Exception:
         # Harden against an inspect failure, which can occur with
         # SWIG-wrapped extensions.
+        raise
         return None
 
 
@@ -143,7 +173,8 b' def getsource(obj,is_binary=False):'
         except TypeError:
             if hasattr(obj,'__class__'):
                 src = inspect.getsource(obj.__class__)
-        return src
+        encoding = get_encoding(obj)
+        return cast_unicode(src, encoding=encoding)
 
 def getargspec(obj):
     """Get the names and default values of a function's arguments.
@@ -319,9 +350,8 b' class Inspector:'
         exception is suppressed."""
 
         try:
-            # We need a plain string here, NOT unicode!
             hdef = oname + inspect.formatargspec(*getargspec(obj))
-            return py3compat.unicode_to_str(hdef, 'ascii')
+            return cast_unicode(hdef)
         except:
             return None
 
@@ -435,7 +465,7 b' class Inspector:'
         except:
             self.noinfo('source',oname)
         else:
-            page.page(self.format(py3compat.unicode_to_str(src)))
+            page.page(self.format(src))
 
     def pfile(self, obj, oname=''):
         """Show the whole file where an object was defined."""
@@ -457,7 +487,7 b' class Inspector:'
             # Print only text files, not extension binaries.  Note that
             # getsourcelines returns lineno with 1-offset and page() uses
             # 0-offset, so we must adjust.
-            page.page(self.format(open(ofile).read()), lineno-1)
+            page.page(self.format(openpy.read_py_file(ofile, skip_encoding_cookie=False)), lineno - 1)
 
     def _format_fields(self, fields, title_width=12):
         """Formats a list of fields for display.
@@ -476,7 +506,7 b' class Inspector:'
                 title = header(title + ":") + "\n"
             else:
                 title = header((title+":").ljust(title_width))
-            out.append(title + content)
+            out.append(cast_unicode(title) + cast_unicode(content))
         return "\n".join(out)
 
     # The fields to be displayed by pinfo: (fancy_name, key_in_info_dict)
@@ -536,7 +566,8 b' class Inspector:'
         # Source or docstring, depending on detail level and whether
         # source found.
         if detail_level > 0 and info['source'] is not None:
-            displayfields.append(("Source", self.format(py3compat.cast_bytes_py2(info['source']))))
+            displayfields.append(("Source", 
+                                  self.format(cast_unicode(info['source']))))
         elif info['docstring'] is not None:
             displayfields.append(("Docstring", info["docstring"]))
 

IPython/core/tests/test_ultratb.py

@@ -1,6 +1,7 b''
+# encoding: utf-8
 """Tests for IPython.core.ultratb
 """
-
+import io
 import os.path
 import unittest
 
@@ -49,3 +50,26 b' class ChangedPyFileTest(unittest.TestCase):'
                     ip.run_cell("foo.f()")
                 with tt.AssertPrints("ZeroDivisionError"):
                     ip.run_cell("foo.f()")
+
+iso_8859_5_file = u'''# coding: iso-8859-5
+
+def fail():
+    """дбИЖ"""
+    1/0     # дбИЖ
+'''
+
+class NonAsciiTest(unittest.TestCase):
+    def test_iso8859_5(self):
+        # Non-ascii directory name as well.
+        with TemporaryDirectory(suffix=u'é') as td:
+            fname = os.path.join(td, 'dfghjkl.py')
+
+            with io.open(fname, 'w', encoding='iso-8859-5') as f:
+                f.write(iso_8859_5_file)
+            
+            with prepended_to_syspath(td):
+                ip.run_cell("from dfghjkl import fail")
+            
+            with tt.AssertPrints("ZeroDivisionError"):
+                with tt.AssertPrints(u'дбИЖ', suppress=False):
+                    ip.run_cell('fail()')

IPython/core/ultratb.py

@@ -69,7 +69,7 b' possible inclusion in future releases.'
 #  the file COPYING, distributed as part of this software.
 #*****************************************************************************
 
-from __future__ import with_statement
+from __future__ import unicode_literals
 
 import inspect
 import keyword
@@ -99,9 +99,12 b' from IPython.core.display_trap import DisplayTrap'
 from IPython.core.excolors import exception_colors
 from IPython.utils import PyColorize
 from IPython.utils import io
+from IPython.utils import path as util_path
 from IPython.utils import py3compat
 from IPython.utils import pyfile
+from IPython.utils import ulinecache
 from IPython.utils.data import uniq_stable
+from IPython.utils.openpy import read_py_file
 from IPython.utils.warn import info, error
 
 # Globals
@@ -229,7 +232,6 b' def fix_frame_records_filenames(records):'
 
 
 def _fixed_getinnerframes(etb, context=1,tb_offset=0):
-    import linecache
     LNUM_POS, LINES_POS, INDEX_POS =  2, 4, 5
 
     records  = fix_frame_records_filenames(inspect.getinnerframes(etb, context))
@@ -251,7 +253,7 b' def _fixed_getinnerframes(etb, context=1,tb_offset=0):'
         maybeStart = lnum-1 - context//2
         start =  max(maybeStart, 0)
         end   = start + context
-        lines = linecache.getlines(file)[start:end]
+        lines = ulinecache.getlines(file)[start:end]
         buf = list(records[i])
         buf[LNUM_POS] = lnum
         buf[INDEX_POS] = lnum - 1 - start
@@ -282,12 +284,7 b' def _format_traceback_lines(lnum, index, lines, Colors, lvals=None,scheme=None):'
     _line_format = _parser.format2
 
     for line in lines:
-        # FIXME: we need to ensure the source is a pure string at this point,
-        # else the coloring code makes a  royal mess.  This is in need of a
-        # serious refactoring, so that all of the ultratb and PyColorize code
-        # is unicode-safe.  So for now this is rather an ugly hack, but
-        # necessary to at least have readable tracebacks. Improvements welcome!
-        line = py3compat.cast_bytes_py2(line, 'utf-8')
+        line = py3compat.cast_unicode(line)
 
         new_line, err = _line_format(line, 'str', scheme)
         if not err: line = new_line
@@ -547,14 +544,13 b' class ListTB(TBTools):'
 
         Also lifted nearly verbatim from traceback.py
         """
-
         have_filedata = False
         Colors = self.Colors
         list = []
         stype = Colors.excName + etype.__name__ + Colors.Normal
         if value is None:
             # Not sure if this can still happen in Python 2.6 and above
-            list.append( str(stype) + '\n')
+            list.append( py3compat.cast_unicode(stype) + '\n')
         else:
             if etype is SyntaxError:
                 have_filedata = True
@@ -562,18 +558,22 b' class ListTB(TBTools):'
                 if not value.filename: value.filename = "<string>"
                 list.append('%s  File %s"%s"%s, line %s%d%s\n' % \
                         (Colors.normalEm,
-                         Colors.filenameEm, value.filename, Colors.normalEm,
+                         Colors.filenameEm, py3compat.cast_unicode(value.filename), Colors.normalEm,
                          Colors.linenoEm, value.lineno, Colors.Normal  ))
-                if value.text is not None:
+                textline = ulinecache.getline(value.filename, value.lineno)
+                if textline == '':
+                    textline = py3compat.cast_unicode(value.text, "utf-8")
+
+                if textline is not None:
                     i = 0
-                    while i < len(value.text) and value.text[i].isspace():
+                    while i < len(textline) and textline[i].isspace():
                         i += 1
                     list.append('%s    %s%s\n' % (Colors.line,
-                                                  value.text.strip(),
+                                                  textline.strip(),
                                                   Colors.Normal))
                     if value.offset is not None:
                         s = '    '
-                        for c in value.text[i:value.offset-1]:
+                        for c in textline[i:value.offset-1]:
                             if c.isspace():
                                 s += c
                             else:
@@ -779,10 +779,9 b' class VerboseTB(TBTools):'
         abspath = os.path.abspath
         for frame, file, lnum, func, lines, index in records:
             #print '*** record:',file,lnum,func,lines,index  # dbg
-
             if not file:
                 file = '?'
-            elif not(file.startswith("<") and file.endswith(">")):
+            elif not(file.startswith(str("<")) and file.endswith(str(">"))):
                 # Guess that filenames like <string> aren't real filenames, so
                 # don't call abspath on them.                    
                 try:
@@ -791,7 +790,7 b' class VerboseTB(TBTools):'
                     # Not sure if this can still happen: abspath now works with
                     # file names like <string>
                     pass
-
+            file = py3compat.cast_unicode(file, util_path.fs_encoding)
             link = tpl_link % file
             args, varargs, varkw, locals = inspect.getargvalues(frame)
 
@@ -831,7 +830,7 b' class VerboseTB(TBTools):'
                 # Look up the corresponding source file.
                 file = pyfile.source_from_cache(file)
 
-            def linereader(file=file, lnum=[lnum], getline=linecache.getline):
+            def linereader(file=file, lnum=[lnum], getline=ulinecache.getline):
                 line = getline(file, lnum[0])
                 lnum[0] += 1
                 return line
@@ -926,7 +925,7 b' class VerboseTB(TBTools):'
             etype_str,evalue_str = map(str,(etype,evalue))
         # ... and format it
         exception = ['%s%s%s: %s' % (Colors.excName, etype_str,
-                                     ColorsNormal, evalue_str)]
+                                     ColorsNormal, py3compat.cast_unicode(evalue_str))]
         if (not py3compat.PY3) and type(evalue) is types.InstanceType:
             try:
                 names = [w for w in dir(evalue) if isinstance(w, basestring)]
@@ -938,7 +937,7 b' class VerboseTB(TBTools):'
                 exception.append(_m % (Colors.excName,ColorsNormal))
                 etype_str,evalue_str = map(str,sys.exc_info()[:2])
                 exception.append('%s%s%s: %s' % (Colors.excName,etype_str,
-                                     ColorsNormal, evalue_str))
+                                     ColorsNormal, py3compat.cast_unicode(evalue_str)))
                 names = []
             for name in names:
                 value = text_repr(getattr(evalue, name))

IPython/lib/clipboard.py

@@ -5,7 +5,7 b' import subprocess'
 import sys
 
 from IPython.core.error import TryNext
-
+import IPython.utils.py3compat as py3compat
 
 def win32_clipboard_get():
     """ Get the current clipboard's text on Windows.
@@ -31,6 +31,7 b' def osx_clipboard_get():'
     text, stderr = p.communicate()
     # Text comes in with old Mac \r line endings. Change them to \n.
     text = text.replace('\r', '\n')
+    text = py3compat.cast_unicode(text, py3compat.DEFAULT_ENCODING)
     return text
 
 def tkinter_clipboard_get():
@@ -49,6 +50,7 b' def tkinter_clipboard_get():'
     root.withdraw()
     text = root.clipboard_get()
     root.destroy()
+    text = py3compat.cast_unicode(text, py3compat.DEFAULT_ENCODING)
     return text
 
 

IPython/lib/demo.py

@@ -1,3 +1,6 b''
+from __future__ import unicode_literals
+
+
 """Module for interactive demos using IPython.
 
 This module implements a few classes for running Python scripts interactively
@@ -179,7 +182,7 b' from IPython.utils.PyColorize import Parser'
 from IPython.utils import io
 from IPython.utils.io import file_read, file_readlines
 from IPython.utils.text import marquee
-
+from IPython.utils import openpy
 __all__ = ['Demo','IPythonDemo','LineDemo','IPythonLineDemo','DemoError']
 
 class DemoError(Exception): pass
@@ -264,13 +267,13 b' class Demo(object):'
             self.fobj = self.src
         else:
              # Assume it's a string or something that can be converted to one
-            self.fobj = open(self.fname)
+            self.fobj = openpy.open(self.fname)
 
     def reload(self):
         """Reload source from disk and initialize state."""
         self.fload()
 
-        self.src     = self.fobj.read()
+        self.src     = "".join(openpy.strip_encoding_cookie(self.fobj))
         src_b        = [b.strip() for b in self.re_stop.split(self.src) if b]
         self._silent = [bool(self.re_silent.findall(b)) for b in src_b]
         self._auto   = [bool(self.re_auto.findall(b)) for b in src_b]

IPython/utils/PyColorize.py

@@ -30,6 +30,8 b' formatting (which is the hard part).'
 """
 from __future__ import print_function
 
+from __future__ import unicode_literals
+
 __all__ = ['ANSICodeColors','Parser']
 
 _scheme_default = 'Linux'

IPython/utils/openpy.py

@@ -7,7 +7,7 b' Much of the code is taken from the tokenize module in Python 3.2.'
 from __future__ import absolute_import
 
 import io
-from io import TextIOWrapper
+from io import TextIOWrapper, BytesIO
 import re
 import urllib
 
@@ -120,6 +120,32 b' except ImportError:'
         text.mode = 'r'
         return text   
 
+def source_to_unicode(txt, errors='replace', skip_encoding_cookie=True):
+    """Converts a bytes string with python source code to unicode.
+
+    Unicode strings are passed through unchanged. Byte strings are checked
+    for the python source file encoding cookie to determine encoding.
+    txt can be either a bytes buffer or a string containing the source
+    code.
+    """
+    if isinstance(txt, unicode):
+        return txt
+    if isinstance(txt, bytes):
+        buffer = BytesIO(txt)
+    else:
+        buffer = txt
+    try:
+        encoding, _ = detect_encoding(buffer.readline)
+    except SyntaxError:
+        encoding = "ascii"
+    buffer.seek(0)
+    text = TextIOWrapper(buffer, encoding, errors=errors, line_buffering=True)
+    text.mode = 'r'
+    if skip_encoding_cookie:
+        return u"".join(strip_encoding_cookie(text))
+    else:
+        return text.read()
+
 def strip_encoding_cookie(filelike):
     """Generator to pull lines from a text-mode file, skipping the encoding
     cookie if it is found in the first two lines.
@@ -181,12 +207,13 b" def read_py_url(url, errors='replace', skip_encoding_cookie=True):"
     """
     response = urllib.urlopen(url)
     buffer = io.BytesIO(response.read())
-    encoding, lines = detect_encoding(buffer.readline)
-    buffer.seek(0)
-    text = TextIOWrapper(buffer, encoding, errors=errors, line_buffering=True)
-    text.mode = 'r'
-    if skip_encoding_cookie:
-        return "".join(strip_encoding_cookie(text))
-    else:
-        return text.read()
+    return source_to_unicode(buffer, errors, skip_encoding_cookie)
 
+def _list_readline(x):
+    """Given a list, returns a readline() function that returns the next element
+    with each call.
+    """
+    x = iter(x)
+    def readline():
+        return next(x)
+    return readline

IPython/zmq/zmqshell.py

@@ -38,7 +38,7 b' from IPython.lib.kernel import ('
     get_connection_file, get_connection_info, connect_qtconsole
 )
 from IPython.testing.skipdoctest import skip_doctest
-from IPython.utils import io
+from IPython.utils import io, openpy
 from IPython.utils.jsonutil import json_clean, encode_images
 from IPython.utils.process import arg_split
 from IPython.utils import py3compat
@@ -355,7 +355,9 b' class KernelMagics(Magics):'
 
         cont = open(arg_s).read()
         if arg_s.endswith('.py'):
-            cont = self.shell.pycolorize(cont)
+            cont = self.shell.pycolorize(openpy.read_py_file(arg_s, skip_encoding_cookie=False))
+        else:
+            cont = open(arg_s).read()
         page.page(cont)
 
     more = line_magic('more')(less)

docs/examples/lib/example-demo.py

@@ -1,9 +1,12 b''
+# -*- coding: utf-8 -*-
 """A simple interactive demo to illustrate the use of IPython's Demo class.
 
 Any python script can be run as a demo, but that does little more than showing
 it on-screen, syntax-highlighted in one shot.  If you add a little simple
 markup, you can stop at specified intervals and return to the ipython prompt,
 resuming execution later.
+
+This is a unicode test, åäö
 """
 from __future__ import print_function