From d1c8952f2cb4a3ded23e169e1260ace1691bee99 2012-08-27 06:03:31
From: Jörgen Stenarson <jorgen.stenarson@kroywen.se>
Date: 2012-08-27 06:03:31
Subject: [PATCH] determine encoding of source lines in %debug before colorizing

---

diff --git a/IPython/core/debugger.py b/IPython/core/debugger.py
index fc8ac88..dd72df5 100644
--- a/IPython/core/debugger.py
+++ b/IPython/core/debugger.py
@@ -352,6 +352,7 @@ class Pdb(OldPdb):
 
         start = lineno - 1 - context//2
         lines = linecache.getlines(filename)
+        encoding = io.guess_encoding(lines)
         start = max(start, 0)
         start = min(start, len(lines) - context)
         lines = lines[start : start + context]
@@ -362,9 +363,8 @@ class Pdb(OldPdb):
                       and tpl_line_em \
                       or tpl_line
             ret.append(self.__format_line(linetpl, filename,
-                                          start + 1 + i, line,
+                                          start + 1 + i, line.decode(encoding),
                                           arrow = show_arrow) )
-
         return ''.join(ret)
 
     def __format_line(self, tpl_line, filename, lineno, line, arrow = False):
@@ -422,8 +422,10 @@ class Pdb(OldPdb):
             tpl_line = '%%s%s%%s %s%%s' % (Colors.lineno, ColorsNormal)
             tpl_line_em = '%%s%s%%s %s%%s%s' % (Colors.linenoEm, Colors.line, ColorsNormal)
             src = []
+            lines = linecache.getlines(filename)
+            encoding = io.guess_encoding(lines)
             for lineno in range(first, last+1):
-                line = linecache.getline(filename, lineno)
+                line = lines[lineno].decode(encoding)
                 if not line:
                     break
 
diff --git a/IPython/utils/io.py b/IPython/utils/io.py
index 40a3b96..3b9e0eb 100644
--- a/IPython/utils/io.py
+++ b/IPython/utils/io.py
@@ -155,20 +155,27 @@ class Tee(object):
             self.close()
 
 
-def source_to_unicode(txt):
-    """Converts string with python source code to unicode
-    """
-    if isinstance(txt, unicode):
-        return txt
+def guess_encoding(lines):
+    """check list of lines for line matching the source code encoding pattern
     
+    Only check first two lines
+    """
     reg = re.compile("#.*coding[:=]\s*([-\w.]+)")
-    for row in txt.split("\n", 2)[:2]: #We only need to check the first two lines
+    for row in lines[:2]: #We only need to check the first two lines
         result = reg.match(row)
         if result:
             coding = result.groups()[0]
             break
     else:
         coding = "ascii"
+    return coding
+
+def source_to_unicode(txt):
+    """Converts string with python source code to unicode
+    """
+    if isinstance(txt, unicode):
+        return txt
+    coding = guess_encoding(txt.split("\n", 2))
     return txt.decode(coding, errors="replace")