Merge pull request #913 from takluyver/py3-tests2

Fix more tests under Python3.  The test suite isn't fully fixed yet under py3, but this gets us closer.

IPython/core/oinspect.py

@@ -31,6 +31,7 b' except ImportError:'
 
 # IPython's own
 from IPython.core import page
+from IPython.testing.skipdoctest import skip_doctest_py3
 from IPython.utils import PyColorize
 from IPython.utils import io
 from IPython.utils import py3compat
@@ -297,6 +298,8 b' class Inspector:'
         else:
             print >>io.stdout, header,self.format(output),
 
+    # In Python 3, all classes are new-style, so they all have __init__.
+    @skip_doctest_py3
     def pdoc(self,obj,oname='',formatter = None):
         """Print the docstring for any object.
 

IPython/core/tests/test_interactiveshell.py

@@ -219,4 +219,4 b' class TestSafeExecfileNonAsciiPath(unittest.TestCase):'
     def test_1(self):
         """Test safe_execfile with non-ascii path
         """
-        _ip.shell.safe_execfile(self.fname, raise_exceptions=True)
+        _ip.shell.safe_execfile(self.fname, {}, raise_exceptions=True)

IPython/core/tests/test_magic.py

@@ -324,7 +324,9 b' def check_cpaste(code, should_fail=False):'
     _ip.user_ns['code_ran'] = False
 
     src = StringIO()
-    src.encoding = None   # IPython expects stdin to have an encoding attribute
+    if not hasattr(src, 'encoding'):
+        # IPython expects stdin to have an encoding attribute
+        src.encoding = None
     src.write('\n')
     src.write(code)
     src.write('\n--\n')

IPython/extensions/tests/test_autoreload.py

@@ -11,6 +11,7 b' import IPython.testing.tools as tt'
 
 from IPython.extensions.autoreload import AutoreloadInterface
 from IPython.core.hooks import TryNext
+from IPython.testing.decorators import knownfailureif
 
 #-----------------------------------------------------------------------------
 # Test fixture
@@ -293,8 +294,12 b' x = -99'
         self.shell.run_code("pass") # trigger reload
         nt.assert_equal(mod.x, -99)
 
+    # The autoreload extension needs to be updated for Python 3.2, as .pyc files
+    # are stored in a different location. See gh-846.
+    @knownfailureif(sys.version_info >= (3,2))
     def test_smoketest_aimport(self):
         self._check_smoketest(use_aimport=True)
 
+    @knownfailureif(sys.version_info >= (3,2))
     def test_smoketest_autoreload(self):
         self._check_smoketest(use_aimport=False)

IPython/lib/tests/test_irunner.py

@@ -13,6 +13,7 b' import unittest'
 
 # IPython imports
 from IPython.lib import irunner
+from IPython.testing.decorators import known_failure_py3
 
 # Testing code begins
 class RunnerTestCase(unittest.TestCase):
@@ -56,6 +57,8 b' class RunnerTestCase(unittest.TestCase):'
         self.assert_(mismatch==0,'Number of mismatched lines: %s' %
                      mismatch)
 
+    # irunner isn't working on Python 3 (due to pexpect)
+    @known_failure_py3
     def testIPython(self):
         """Test the IPython runner."""
         source = """
@@ -129,6 +132,7 b' In [12]: exit'
         runner = irunner.IPythonRunner(out=self.out)
         self._test_runner(runner,source,output)
 
+    @known_failure_py3
     def testPython(self):
         """Test the Python runner."""
         runner = irunner.PythonRunner(out=self.out)

IPython/lib/tests/test_irunner_pylab_magic.py

@@ -22,6 +22,7 b' class RunnerTestCase(unittest.TestCase):'
         self.out = StringIO.StringIO()
         #self.out = sys.stdout
 
+    @decorators.known_failure_py3
     def _test_runner(self,runner,source,output):
         """Test that a given runner's input/output match."""
 
@@ -82,6 +83,7 b' Out\\[6\\]: True'
         runner = irunner.IPythonRunner(out=self.out)
         self._test_runner(runner,source,output)
 
+    @decorators.known_failure_py3
     @decorators.skipif_not_matplotlib
     def test_pylab_import_all_disabled(self):
         "Verify that plot is not available when pylab_import_all = False"

IPython/nbformat/v2/nbjson.py

@@ -16,9 +16,8 b' Authors:'
 # Imports
 #-----------------------------------------------------------------------------
 
-from base64 import encodestring
 from .nbbase import from_dict
-from .rwbase import NotebookReader, NotebookWriter, base64_decode
+from .rwbase import NotebookReader, NotebookWriter, restore_bytes
 import json
 
 #-----------------------------------------------------------------------------
@@ -26,9 +25,10 b' import json'
 #-----------------------------------------------------------------------------
 
 class BytesEncoder(json.JSONEncoder):
+    """A JSON encoder that accepts b64 (and other *ascii*) bytestrings."""
     def default(self, obj):
         if isinstance(obj, bytes):
-            return unicode(encodestring(bytes))
+            return obj.decode('ascii')
         return json.JSONEncoder.default(self, obj)
 
 
@@ -40,7 +40,7 b' class JSONReader(NotebookReader):'
         return nb
 
     def to_notebook(self, d, **kwargs):
-        return base64_decode(from_dict(d))
+        return restore_bytes(from_dict(d))
 
 
 class JSONWriter(NotebookWriter):

IPython/nbformat/v2/rwbase.py

@@ -19,31 +19,67 b' Authors:'
 from base64 import encodestring, decodestring
 import pprint
 
+from IPython.utils.py3compat import str_to_bytes
+
 #-----------------------------------------------------------------------------
 # Code
 #-----------------------------------------------------------------------------
 
+def restore_bytes(nb):
+    """Restore bytes of image data from unicode-only formats.
+    
+    Base64 encoding is handled elsewhere.  Bytes objects in the notebook are
+    always b64-encoded. We DO NOT encode/decode around file formats.
+    """
+    for ws in nb.worksheets:
+        for cell in ws.cells:
+            if cell.cell_type == 'code':
+                for output in cell.outputs:
+                    if 'png' in output:
+                        output.png = str_to_bytes(output.png, 'ascii')
+                    if 'jpeg' in output:
+                        output.jpeg = str_to_bytes(output.jpeg, 'ascii')
+    return nb
+
+
+# b64 encode/decode are never actually used, because all bytes objects in
+# the notebook are already b64-encoded, and we don't need/want to double-encode
+
 def base64_decode(nb):
-    """Base64 encode all bytes objects in the notebook."""
+    """Restore all bytes objects in the notebook from base64-encoded strings.
+    
+    Note: This is never used
+    """
     for ws in nb.worksheets:
         for cell in ws.cells:
             if cell.cell_type == 'code':
-                if 'png' in cell:
-                    cell.png = bytes(decodestring(cell.png))
-                if 'jpeg' in cell:
-                    cell.jpeg = bytes(decodestring(cell.jpeg))
+                for output in cell.outputs:
+                    if 'png' in output:
+                        if isinstance(output.png, unicode):
+                            output.png = output.png.encode('ascii')
+                        output.png = decodestring(output.png)
+                    if 'jpeg' in output:
+                        if isinstance(output.jpeg, unicode):
+                            output.jpeg = output.jpeg.encode('ascii')
+                        output.jpeg = decodestring(output.jpeg)
     return nb
 
 
 def base64_encode(nb):
-    """Base64 decode all binary objects in the notebook."""
+    """Base64 encode all bytes objects in the notebook.
+    
+    These will be b64-encoded unicode strings
+    
+    Note: This is never used
+    """
     for ws in nb.worksheets:
         for cell in ws.cells:
             if cell.cell_type == 'code':
-                if 'png' in cell:
-                    cell.png = unicode(encodestring(cell.png))
-                if 'jpeg' in cell:
-                    cell.jpeg = unicode(encodestring(cell.jpeg))
+                for output in cell.outputs:
+                    if 'png' in output:
+                        output.png = encodestring(output.png).decode('ascii')
+                    if 'jpeg' in output:
+                        output.jpeg = encodestring(output.jpeg).decode('ascii')
     return nb
 
 

IPython/nbformat/v2/tests/nbexamples.py

@@ -1,10 +1,15 b''
+import os
+from base64 import encodestring
+
 from ..nbbase import (
     NotebookNode,
     new_code_cell, new_text_cell, new_worksheet, new_notebook, new_output,
     new_metadata, new_author
 )
 
-
+# some random base64-encoded *bytes*
+png = encodestring(os.urandom(5))
+jpeg = encodestring(os.urandom(6))
 
 ws = new_worksheet(name='worksheet1')
 
@@ -42,8 +47,8 b' ws.cells.append(new_code_cell('
         output_text=u'<array a>',
         output_html=u'The HTML rep',
         output_latex=u'$a$',
-        output_png=b'data',
-        output_jpeg=b'data',
+        output_png=png,
+        output_jpeg=jpeg,
         output_svg=u'<svg>',
         output_json=u'json data',
         output_javascript=u'var i=0;',
@@ -53,8 +58,8 b' ws.cells.append(new_code_cell('
         output_text=u'<array a>',
         output_html=u'The HTML rep',
         output_latex=u'$a$',
-        output_png=b'data',
-        output_jpeg=b'data',
+        output_png=png,
+        output_jpeg=jpeg,
         output_svg=u'<svg>',
         output_json=u'json data',
         output_javascript=u'var i=0;'

IPython/testing/decorators.py

@@ -323,6 +323,9 b" skipif_not_sympy = skip_without('sympy')"
 
 skip_known_failure = knownfailureif(True,'This test is known to fail')
 
+known_failure_py3 = knownfailureif(sys.version_info[0] >= 3, 
+                                    'This test is known to fail on Python 3.')
+
 # A null 'decorator', useful to make more readable code that needs to pick
 # between different decorators based on OS or other conditions
 null_deco = lambda f: f

IPython/testing/skipdoctest.py

@@ -4,6 +4,7 b' The IPython.testing.decorators module triggers various extra imports, including'
 numpy and sympy if they're present. Since this decorator is used in core parts
 of IPython, it's in a separate module so that running IPython doesn't trigger
 those imports."""
+import sys
 
 def skip_doctest(f):
     """Decorator - mark a function or method for skipping its doctest.
@@ -13,3 +14,8 b' def skip_doctest(f):'
     etc."""
     f.skip_doctest = True
     return f
+
+def skip_doctest_py3(f):
+    """Decorator - skip the doctest under Python 3."""
+    f.skip_doctest = (sys.version_info[0] >= 3)
+    return f