Merge pull request #1480 from minrk/npmagic

Fix %notebook magic, etc. nbformat unicode tests and fixes.

* json.writes always gives unicode, so that `current.writes` can be trusted to give the same interface
* setup base TestCase for nbformat tests, to consolidate code, and better test both file formats
* add tests for reading/writing to files
* allow `name` as kwarg to new_notebook to avoid unnecessary breakage of previous API.
* remove fallback to xml, which would hide corrupt notebook files behind a nonsensical 'xml unsupported' message.

Closes  #1545,  #1487.

IPython/nbformat/v3/tests/formattest.py

@@ -0,0 +1,63
+# -*- coding: utf8 -*-
+import io
+import os
+import shutil
+import tempfile
+
+pjoin = os.path.join
+
+from ..nbbase import (
+    NotebookNode,
+    new_code_cell, new_text_cell, new_worksheet, new_notebook
+)
+
+from ..nbpy import reads, writes, read, write
+from .nbexamples import nb0, nb0_py
+
+
+def open_utf8(fname, mode):
+    return io.open(fname, mode=mode, encoding='utf-8')
+
+class NBFormatTest:
+    """Mixin for writing notebook format tests"""
+
+    # override with appropriate values in subclasses
+    nb0_ref = None
+    ext = None
+    mod = None
+    
+    def setUp(self):
+        self.wd = tempfile.mkdtemp()
+    
+    def tearDown(self):
+        shutil.rmtree(self.wd)
+    
+    def assertNBEquals(self, nba, nbb):
+        self.assertEquals(nba, nbb)
+        
+    def test_writes(self):
+        s = self.mod.writes(nb0)
+        if self.nb0_ref:
+            self.assertEquals(s, self.nb0_ref)
+
+    def test_reads(self):
+        s = self.mod.writes(nb0)
+        nb = self.mod.reads(s)
+
+    def test_roundtrip(self):
+        s = self.mod.writes(nb0)
+        self.assertNBEquals(self.mod.reads(s),nb0)
+
+    def test_write_file(self):
+        with open_utf8(pjoin(self.wd, "nb0.%s" % self.ext), 'w') as f:
+            self.mod.write(nb0, f)
+    
+    def test_read_file(self):
+        with open_utf8(pjoin(self.wd, "nb0.%s" % self.ext), 'w') as f:
+            self.mod.write(nb0, f)
+        
+        with open_utf8(pjoin(self.wd, "nb0.%s" % self.ext), 'r') as f:
+            nb = self.mod.read(f)
+        
+
+

IPython/core/magic.py

@@ -20,6 +20,7 import __future__
 import bdb
 import inspect
 import imp
+import io
 import os
 import sys
 import shutil
@@ -2222,7 +2223,7 Currently the magic system has the following functions:\n"""
         except (TypeError, ValueError) as e:
             print e.args[0]
             return
-        with py3compat.open(fname,'w', encoding="utf-8") as f:
+        with io.open(fname,'w', encoding="utf-8") as f:
             f.write(u"# coding: utf-8\n")
             f.write(py3compat.cast_unicode(cmds))
         print 'The following commands were written to file `%s`:' % fname
@@ -3663,7 +3664,7 Defaulting color scheme to 'NoColor'"""
                 cells.append(current.new_code_cell(prompt_number=prompt_number, input=input))
             worksheet = current.new_worksheet(cells=cells)
             nb = current.new_notebook(name=name,worksheets=[worksheet])
-            with open(fname, 'w') as f:
+            with io.open(fname, 'w', encoding='utf-8') as f:
                 current.write(nb, f, format);
         elif args.format is not None:
             old_fname, old_name, old_format = current.parse_filename(args.filename)
@@ -3677,13 +3678,9 Defaulting color scheme to 'NoColor'"""
                 new_fname = old_name + u'.py'
             else:
                 raise ValueError('Invalid notebook format: %s' % new_format)
-            with open(old_fname, 'r') as f:
-                s = f.read()
-                try:
-                    nb = current.reads(s, old_format)
-                except:
-                    nb = current.reads(s, u'xml')
-            with open(new_fname, 'w') as f:
+            with io.open(old_fname, 'r', encoding='utf-8') as f:
+                nb = current.read(f, old_format)
+            with io.open(new_fname, 'w', encoding='utf-8') as f:
                 current.write(nb, f, new_format)
 
     def magic_config(self, s):

IPython/core/tests/test_magic.py

@@ -1,3 +1,4
+# -*- coding: utf-8 -*-
 """Tests for various magic functions.
 
 Needs to be run by nose (to make ipython session available).
@@ -8,12 +9,15 from __future__ import absolute_import
 # Imports
 #-----------------------------------------------------------------------------
 
+import io
 import os
 import sys
 from StringIO import StringIO
 
 import nose.tools as nt
 
+from IPython.nbformat.v3.tests.nbexamples import nb0
+from IPython.nbformat import current
 from IPython.testing import decorators as dec
 from IPython.testing import tools as tt
 from IPython.utils import py3compat
@@ -23,6 +27,7 from IPython.utils.tempdir import TemporaryDirectory
 # Test functions begin
 #-----------------------------------------------------------------------------
 
+
 def test_rehashx():
     # clear up everything
     _ip = get_ipython()
@@ -431,3 +436,34 def test_extension():
     finally:
         _ip.ipython_dir = orig_ipython_dir
         
+def test_notebook_export_json():
+    with TemporaryDirectory() as td:
+        outfile = os.path.join(td, "nb.ipynb")
+        _ip.ex(py3compat.u_format(u"u = {u}'héllo'"))
+        _ip.magic("notebook -e %s" % outfile)
+
+def test_notebook_export_py():
+    with TemporaryDirectory() as td:
+        outfile = os.path.join(td, "nb.py")
+        _ip.ex(py3compat.u_format(u"u = {u}'héllo'"))
+        _ip.magic("notebook -e %s" % outfile)
+
+def test_notebook_reformat_py():
+    with TemporaryDirectory() as td:
+        infile = os.path.join(td, "nb.ipynb")
+        with io.open(infile, 'w') as f:
+            current.write(nb0, f, 'json')
+            
+        _ip.ex(py3compat.u_format(u"u = {u}'héllo'"))
+        _ip.magic("notebook -f py %s" % infile)
+
+def test_notebook_reformat_json():
+    with TemporaryDirectory() as td:
+        infile = os.path.join(td, "nb.py")
+        with io.open(infile, 'w') as f:
+            current.write(nb0, f, 'py')
+            
+        _ip.ex(py3compat.u_format(u"u = {u}'héllo'"))
+        _ip.magic("notebook -f ipynb %s" % infile)
+        _ip.magic("notebook -f json %s" % infile)
+

IPython/nbformat/v3/nbbase.py

@@ -146,7 +146,7 def new_worksheet(name=None, cells=None):
     return ws
 
 
-def new_notebook(metadata=None, worksheets=None):
+def new_notebook(name=None, metadata=None, worksheets=None):
     """Create a notebook by name, id and a list of worksheets."""
     nb = NotebookNode()
     nb.nbformat = nbformat
@@ -158,6 +158,8 def new_notebook(metadata=None, worksheets=None):
         nb.metadata = new_metadata()
     else:
         nb.metadata = NotebookNode(metadata)
+    if name is not None:
+        nb.metadata.name = unicode(name)
     return nb
 
 

IPython/nbformat/v3/nbjson.py

@@ -24,6 +24,8 from .rwbase import (
     NotebookReader, NotebookWriter, restore_bytes, rejoin_lines, split_lines
 )
 
+from IPython.utils import py3compat
+
 #-----------------------------------------------------------------------------
 # Code
 #-----------------------------------------------------------------------------
@@ -56,7 +58,7 class JSONWriter(NotebookWriter):
         kwargs['separators'] = (',',': ')
         if kwargs.pop('split_lines', True):
             nb = split_lines(copy.deepcopy(nb))
-        return json.dumps(nb, **kwargs)
+        return py3compat.str_to_unicode(json.dumps(nb, **kwargs), 'utf-8')
     
 
 _reader = JSONReader()

IPython/nbformat/v3/rwbase.py

@@ -19,7 +19,9 Authors:
 from base64 import encodestring, decodestring
 import pprint
 
-from IPython.utils.py3compat import str_to_bytes
+from IPython.utils import py3compat
+
+str_to_bytes = py3compat.str_to_bytes
 
 #-----------------------------------------------------------------------------
 # Code
@@ -84,17 +86,17 def split_lines(nb):
         for cell in ws.cells:
             if cell.cell_type == 'code':
                 if 'input' in cell and isinstance(cell.input, basestring):
-                    cell.input = cell.input.splitlines()
+                    cell.input = (cell.input + '\n').splitlines()
                 for output in cell.outputs:
                     for key in _multiline_outputs:
                         item = output.get(key, None)
                         if isinstance(item, basestring):
-                            output[key] = item.splitlines()
+                            output[key] = (item + '\n').splitlines()
             else: # text, heading cell
                 for key in ['source', 'rendered']:
                     item = cell.get(key, None)
                     if isinstance(item, basestring):
-                        cell[key] = item.splitlines()
+                        cell[key] = (item + '\n').splitlines()
     return nb
 
 # b64 encode/decode are never actually used, because all bytes objects in
@@ -147,7 +149,10 class NotebookReader(object):
 
     def read(self, fp, **kwargs):
         """Read a notebook from a file like object"""
-        return self.read(fp.read(), **kwargs)
+        nbs = fp.read()
+        if not py3compat.PY3 and not isinstance(nbs, unicode):
+            nbs = py3compat.str_to_unicode(nbs)
+        return self.reads(nbs, **kwargs)
 
 
 class NotebookWriter(object):
@@ -159,7 +164,11 class NotebookWriter(object):
 
     def write(self, nb, fp, **kwargs):
         """Write a notebook to a file like object"""
-        return fp.write(self.writes(nb,**kwargs))
+        nbs = self.writes(nb,**kwargs)
+        if not py3compat.PY3 and not isinstance(nbs, unicode):
+            # this branch is likely only taken for JSON on Python 2
+            nbs = py3compat.str_to_unicode(nbs)
+        return fp.write(nbs)
 
 
 

IPython/nbformat/v3/tests/nbexamples.py

@@ -1,3 +1,5
+# -*- coding: utf-8 -*-
+
 import os
 from base64 import encodestring
 
@@ -47,9 +49,17 ws.cells.append(new_code_cell(
     prompt_number=2,
     collapsed=True
 ))
+ws.cells.append(new_code_cell(
+    input='a = 10\nb = 5\n',
+    prompt_number=3,
+))
+ws.cells.append(new_code_cell(
+    input='a = 10\nb = 5',
+    prompt_number=4,
+))
 
 ws.cells.append(new_code_cell(
-    input='print a',
+    input=u'print "ünîcødé"',
     prompt_number=3,
     collapsed=False,
     outputs=[new_output(
@@ -91,7 +101,7 nb0 = new_notebook(
     metadata=md
 )
 
-nb0_py = """# -*- coding: utf-8 -*-
+nb0_py = u"""# -*- coding: utf-8 -*-
 # <nbformat>%i</nbformat>
 
 # <htmlcell>
@@ -120,7 +130,17 a = numpy.random.rand(100)
 
 # <codecell>
 
-print a
+a = 10
+b = 5
+
+# <codecell>
+
+a = 10
+b = 5
+
+# <codecell>
+
+print "ünîcødé"
 
 """ % nbformat
 

IPython/nbformat/v3/tests/test_json.py

@@ -2,33 +2,32 import pprint
 from unittest import TestCase
 
 from ..nbjson import reads, writes
+from .. import nbjson
 from .nbexamples import nb0
 
+from . import formattest
 
-class TestJSON(TestCase):
+from .nbexamples import nb0
+
+
+class TestJSON(formattest.NBFormatTest, TestCase):
+
+    nb0_ref = None
+    ext = 'ipynb'
+    mod = nbjson
 
-    def test_roundtrip(self):
-        s = writes(nb0)
-        # print
-        # print pprint.pformat(nb0,indent=2)
-        # print
-        # print pprint.pformat(reads(s),indent=2)
-        # print
-        # print s
-        self.assertEquals(reads(s),nb0)
-    
     def test_roundtrip_nosplit(self):
         """Ensure that multiline blobs are still readable"""
         # ensures that notebooks written prior to splitlines change
         # are still readable.
         s = writes(nb0, split_lines=False)
-        self.assertEquals(reads(s),nb0)
+        self.assertEquals(nbjson.reads(s),nb0)
 
     def test_roundtrip_split(self):
         """Ensure that splitting multiline blocks is safe"""
         # This won't differ from test_roundtrip unless the default changes
         s = writes(nb0, split_lines=True)
-        self.assertEquals(reads(s),nb0)
+        self.assertEquals(nbjson.reads(s),nb0)
 
 
 

IPython/nbformat/v3/tests/test_nbbase.py

@@ -112,6 +112,13 class TestNotebook(TestCase):
         self.assertEquals(nb.worksheets,worksheets)
         self.assertEquals(nb.nbformat,nbformat)
 
+    def test_notebook_name(self):
+        worksheets = [new_worksheet(),new_worksheet()]
+        nb = new_notebook(name='foo',worksheets=worksheets)
+        self.assertEquals(nb.metadata.name,u'foo')
+        self.assertEquals(nb.worksheets,worksheets)
+        self.assertEquals(nb.nbformat,nbformat)
+
 class TestMetadata(TestCase):
 
     def test_empty_metadata(self):

IPython/nbformat/v3/tests/test_nbpy.py

@@ -1,17 +1,46
+# -*- coding: utf8 -*-
+
 from unittest import TestCase
 
-from ..nbbase import (
-    NotebookNode,
-    new_code_cell, new_text_cell, new_worksheet, new_notebook
-)
+from . import formattest
 
-from ..nbpy import reads, writes
+from .. import nbpy
 from .nbexamples import nb0, nb0_py
 
 
-class TestPy(TestCase):
+class TestPy(formattest.NBFormatTest, TestCase):
 
-    def test_write(self):
-        s = writes(nb0)
-        self.assertEquals(s,nb0_py)
+    nb0_ref = nb0_py
+    ext = 'py'
+    mod = nbpy
+    ignored_keys = ['collapsed', 'outputs', 'prompt_number', 'metadata']
 
+    def assertSubset(self, da, db):
+        """assert that da is a subset of db, ignoring self.ignored_keys.
+        
+        Called recursively on containers, ultimately comparing individual
+        elements.
+        """
+        if isinstance(da, dict):
+            for k,v in da.iteritems():
+                if k in self.ignored_keys:
+                    continue
+                self.assertTrue(k in db)
+                self.assertSubset(v, db[k])
+        elif isinstance(da, list):
+            for a,b in zip(da, db):
+                self.assertSubset(a,b)
+        else:
+            if isinstance(da, basestring) and isinstance(db, basestring):
+                # pyfile is not sensitive to preserving leading/trailing
+                # newlines in blocks through roundtrip
+                da = da.strip('\n')
+                db = db.strip('\n')
+            self.assertEquals(da, db)
+        return True
+    
+    def assertNBEquals(self, nba, nbb):
+        # since roundtrip is lossy, only compare keys that are preserved
+        # assumes nba is read from my file format
+        return self.assertSubset(nba, nbb)
+        

IPython/utils/py3compat.py

@@ -32,7 +32,7 def cast_bytes(s, encoding=None):
 def _modify_str_or_docstring(str_change_func):
     @functools.wraps(str_change_func)
     def wrapper(func_or_str):
-        if isinstance(func_or_str, str):
+        if isinstance(func_or_str, basestring):
             func = None
             doc = func_or_str
         else: