From 233d50f6f643aef031e0124c92520c04f1fac801 2010-10-11 17:49:32
From: Fernando Perez <Fernando.Perez@berkeley.edu>
Date: 2010-10-11 17:49:32
Subject: [PATCH] Stop-gap fix for crash with unicode input.

The fix is highly suboptimal, as we're still ignoring unicode input.
But at least we don't flat out crash, until I can work on it tomorrow.

---

diff --git a/IPython/core/inputsplitter.py b/IPython/core/inputsplitter.py
index 1f723b7..9267770 100644
--- a/IPython/core/inputsplitter.py
+++ b/IPython/core/inputsplitter.py
@@ -958,6 +958,10 @@ class IPythonInputSplitter(InputSplitter):
         if not lines:
             return super(IPythonInputSplitter, self).push(lines)
 
+        # We must ensure all input is pure unicode
+        if type(lines)==str:
+            lines = lines.decode(self.encoding)
+
         lines_list = lines.splitlines()
 
         transforms = [transform_escaped, transform_assign_system,
diff --git a/IPython/core/interactiveshell.py b/IPython/core/interactiveshell.py
index 790b937..efe053f 100644
--- a/IPython/core/interactiveshell.py
+++ b/IPython/core/interactiveshell.py
@@ -2256,10 +2256,18 @@ class InteractiveShell(Configurable, Magic):
 
         # We need to ensure that the source is unicode from here on.
         if type(source)==str:
-            source = source.decode(self.stdin_encoding)
+            usource = source.decode(self.stdin_encoding)
+        else:
+            usource = source
+
+        if 0:  # dbg
+            print 'Source:', repr(source)  # dbg
+            print 'USource:', repr(usource)  # dbg
+            print 'type:', type(source) # dbg
+            print 'encoding', self.stdin_encoding  # dbg
         
         try:
-            code = self.compile(source,filename,symbol)
+            code = self.compile(usource,filename,symbol)
         except (OverflowError, SyntaxError, ValueError, TypeError, MemoryError):
             # Case 1
             self.showsyntaxerror(filename)
diff --git a/IPython/core/tests/test_inputsplitter.py b/IPython/core/tests/test_inputsplitter.py
index 0bf276f..adc0b4f 100644
--- a/IPython/core/tests/test_inputsplitter.py
+++ b/IPython/core/tests/test_inputsplitter.py
@@ -347,6 +347,10 @@ class InputSplitterTestCase(unittest.TestCase):
         for block_lines in all_blocks:
             self.check_split(block_lines, compile=False)
 
+    def test_unicode(self):
+        self.isp.push(u"Pérez")
+        self.isp.push(u'\xc3\xa9')
+        self.isp.push("u'\xc3\xa9'")
 
 class InteractiveLoopTestCase(unittest.TestCase):
     """Tests for an interactive loop like a python shell.