don't store signatures in notebooks

store them in an sqlite database instead.

The algorithm and signature are stored and not associated with a path.
This means that moving files around doesn't affect trust,
and every trusted change to a notebook remains trusted.

IPython/nbformat/sign.py

@@ -1,331 +1,411 b''
-"""Functions for signing notebooks"""
+"""Utilities for signing notebooks"""
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 import base64
 from contextlib import contextmanager
+from datetime import datetime
 import hashlib
 from hmac import HMAC
 import io
 import os
 
+try:
+    import sqlite3
+except ImportError:
+    try:
+        from pysqlite2 import dbapi2 as sqlite3
+    except ImportError:
+        sqlite3 = None
+
 from IPython.utils.io import atomic_writing
-from IPython.utils.py3compat import string_types, unicode_type, cast_bytes
-from IPython.utils.traitlets import Instance, Bytes, Enum, Any, Unicode, Bool
+from IPython.utils.py3compat import unicode_type, cast_bytes
+from IPython.utils.traitlets import Instance, Bytes, Enum, Any, Unicode, Bool, Integer
 from IPython.config import LoggingConfigurable, MultipleInstanceError
 from IPython.core.application import BaseIPythonApplication, base_flags
 
 from . import read, write, NO_CONVERT
 
 try:
     # Python 3
     algorithms = hashlib.algorithms_guaranteed
 except AttributeError:
     algorithms = hashlib.algorithms
 
 
 def yield_everything(obj):
     """Yield every item in a container as bytes
     
     Allows any JSONable object to be passed to an HMAC digester
     without having to serialize the whole thing.
     """
     if isinstance(obj, dict):
         for key in sorted(obj):
             value = obj[key]
             yield cast_bytes(key)
             for b in yield_everything(value):
                 yield b
     elif isinstance(obj, (list, tuple)):
         for element in obj:
             for b in yield_everything(element):
                 yield b
     elif isinstance(obj, unicode_type):
         yield obj.encode('utf8')
     else:
         yield unicode_type(obj).encode('utf8')
 
 def yield_code_cells(nb):
     """Iterator that yields all cells in a notebook
     
     nbformat version independent
     """
     if nb.nbformat >= 4:
         for cell in nb['cells']:
             if cell['cell_type'] == 'code':
                 yield cell
     elif nb.nbformat == 3:
         for ws in nb['worksheets']:
             for cell in ws['cells']:
                 if cell['cell_type'] == 'code':
                     yield cell
 
 @contextmanager
 def signature_removed(nb):
     """Context manager for operating on a notebook with its signature removed
     
     Used for excluding the previous signature when computing a notebook's signature.
     """
     save_signature = nb['metadata'].pop('signature', None)
     try:
         yield
     finally:
         if save_signature is not None:
             nb['metadata']['signature'] = save_signature
 
 
 class NotebookNotary(LoggingConfigurable):
     """A class for computing and verifying notebook signatures."""
     
     profile_dir = Instance("IPython.core.profiledir.ProfileDir")
     def _profile_dir_default(self):
         from IPython.core.application import BaseIPythonApplication
         app = None
         try:
             if BaseIPythonApplication.initialized():
                 app = BaseIPythonApplication.instance()
         except MultipleInstanceError:
             pass
         if app is None:
             # create an app, without the global instance
             app = BaseIPythonApplication()
             app.initialize(argv=[])
         return app.profile_dir
     
+    db_file = Unicode(config=True)
+    def _db_file_default(self):
+        if self.profile_dir is None:
+            return ':memory:'
+        return os.path.join(self.profile_dir.security_dir, u'nbsignatures.db')
+    
+    # 64k entries ~ 12MB
+    db_size_limit = Integer(65535, config=True)
+    db = Any()
+    def _db_default(self):
+        if sqlite3 is None:
+            self.log.warn("Missing SQLite3, all notebooks will be untrusted!")
+            return
+        kwargs = dict(detect_types=sqlite3.PARSE_DECLTYPES|sqlite3.PARSE_COLNAMES)
+        db = sqlite3.connect(self.db_file, **kwargs)
+        self.init_db(db)
+        return db
+    
+    def init_db(self, db):
+        db.execute("""
+        CREATE TABLE IF NOT EXISTS nbsignatures
+        (
+            id integer PRIMARY KEY AUTOINCREMENT,
+            algorithm text,
+            signature text,
+            path text,
+            last_seen timestamp
+        )""")
+        db.execute("""
+        CREATE INDEX IF NOT EXISTS algosig ON nbsignatures(algorithm, signature)
+        """)
+        db.commit()
+    
     algorithm = Enum(algorithms, default_value='sha256', config=True,
         help="""The hashing algorithm used to sign notebooks."""
     )
     def _algorithm_changed(self, name, old, new):
         self.digestmod = getattr(hashlib, self.algorithm)
     
     digestmod = Any()
     def _digestmod_default(self):
         return getattr(hashlib, self.algorithm)
     
     secret_file = Unicode(config=True,
         help="""The file where the secret key is stored."""
     )
     def _secret_file_default(self):
         if self.profile_dir is None:
             return ''
         return os.path.join(self.profile_dir.security_dir, 'notebook_secret')
     
     secret = Bytes(config=True,
         help="""The secret key with which notebooks are signed."""
     )
     def _secret_default(self):
         # note : this assumes an Application is running
         if os.path.exists(self.secret_file):
             with io.open(self.secret_file, 'rb') as f:
                 return f.read()
         else:
             secret = base64.encodestring(os.urandom(1024))
             self._write_secret_file(secret)
             return secret
     
     def _write_secret_file(self, secret):
         """write my secret to my secret_file"""
         self.log.info("Writing notebook-signing key to %s", self.secret_file)
         with io.open(self.secret_file, 'wb') as f:
             f.write(secret)
         try:
             os.chmod(self.secret_file, 0o600)
         except OSError:
             self.log.warn(
                 "Could not set permissions on %s",
                 self.secret_file
             )
         return secret
     
     def compute_signature(self, nb):
         """Compute a notebook's signature
         
         by hashing the entire contents of the notebook via HMAC digest.
         """
         hmac = HMAC(self.secret, digestmod=self.digestmod)
         # don't include the previous hash in the content to hash
         with signature_removed(nb):
             # sign the whole thing
             for b in yield_everything(nb):
                 hmac.update(b)
         
         return hmac.hexdigest()
     
     def check_signature(self, nb):
         """Check a notebook's stored signature
         
         If a signature is stored in the notebook's metadata,
         a new signature is computed and compared with the stored value.
         
         Returns True if the signature is found and matches, False otherwise.
         
         The following conditions must all be met for a notebook to be trusted:
         - a signature is stored in the form 'scheme:hexdigest'
         - the stored scheme matches the requested scheme
         - the requested scheme is available from hashlib
         - the computed hash from notebook_signature matches the stored hash
         """
         if nb.nbformat < 3:
             return False
-        stored_signature = nb['metadata'].get('signature', None)
-        if not stored_signature \
-            or not isinstance(stored_signature, string_types) \
-            or ':' not in stored_signature:
+        if self.db is None:
             return False
-        stored_algo, sig = stored_signature.split(':', 1)
-        if self.algorithm != stored_algo:
+        signature = self.compute_signature(nb)
+        r = self.db.execute("""SELECT id FROM nbsignatures WHERE
+            algorithm = ? AND
+            signature = ?;
+            """, (self.algorithm, signature)).fetchone()
+        if r is None:
             return False
-        my_signature = self.compute_signature(nb)
-        return my_signature == sig
+        self.db.execute("""UPDATE nbsignatures SET last_seen = ? WHERE
+            algorithm = ? AND
+            signature = ?;
+            """,
+            (datetime.utcnow(), self.algorithm, signature),
+        )
+        self.db.commit()
+        return True
     
     def sign(self, nb):
-        """Sign a notebook, indicating that its output is trusted
-        
-        stores 'algo:hmac-hexdigest' in notebook.metadata.signature
+        """Sign a notebook, indicating that its output is trusted on this machine
         
-        e.g. 'sha256:deadbeef123...'
+        Stores hash algorithm and hmac digest in a local database of trusted notebooks.
         """
         if nb.nbformat < 3:
             return
         signature = self.compute_signature(nb)
-        nb['metadata']['signature'] = "%s:%s" % (self.algorithm, signature)
+        self.store_signature(signature, nb)
+
+    def store_signature(self, signature, nb):
+        if self.db is None:
+            return
+        self.db.execute("""INSERT OR IGNORE INTO nbsignatures
+            (algorithm, signature, last_seen) VALUES (?, ?, ?)""",
+            (self.algorithm, signature, datetime.utcnow())
+        )
+        self.db.execute("""UPDATE nbsignatures SET last_seen = ? WHERE
+            algorithm = ? AND
+            signature = ?;
+            """,
+            (datetime.utcnow(), self.algorithm, signature),
+        )
+        self.db.commit()
+    
+    def unsign(self, nb):
+        """Ensure that a notebook is untrusted
+        
+        by removing its signature from the trusted database, if present.
+        """
+        signature = self.compute_signature(nb)
+        self.db.execute("""DELETE FROM nbsignatures WHERE
+                algorithm = ? AND
+                signature = ?;
+            """,
+            (self.algorithm, signature)
+        )
+        self.db.commit()
+    
+    def cull_db(self):
+        self.db.execute("""DELETE FROM nbsignatures WHERE id IN (
+            SELECT id FROM nbsignatures ORDER BY last_seen DESC LIMIT -1 OFFSET ?
+        );
+        """, (self.db_size_limit,))
     
     def mark_cells(self, nb, trusted):
         """Mark cells as trusted if the notebook's signature can be verified
         
         Sets ``cell.metadata.trusted = True | False`` on all code cells,
         depending on whether the stored signature can be verified.
         
         This function is the inverse of check_cells
         """
         if nb.nbformat < 3:
             return
         
         for cell in yield_code_cells(nb):
             cell['metadata']['trusted'] = trusted
     
     def _check_cell(self, cell, nbformat_version):
         """Do we trust an individual cell?
         
         Return True if:
         
         - cell is explicitly trusted
         - cell has no potentially unsafe rich output
         
         If a cell has no output, or only simple print statements,
         it will always be trusted.
         """
         # explicitly trusted
         if cell['metadata'].pop("trusted", False):
             return True
         
         # explicitly safe output
         if nbformat_version >= 4:
-            safe = {'text/plain', 'image/png', 'image/jpeg'}
             unsafe_output_types = ['execute_result', 'display_data']
             safe_keys = {"output_type", "execution_count", "metadata"}
         else: # v3
-            safe = {'text', 'png', 'jpeg'}
             unsafe_output_types = ['pyout', 'display_data']
             safe_keys = {"output_type", "prompt_number", "metadata"}
         
         for output in cell['outputs']:
             output_type = output['output_type']
             if output_type in unsafe_output_types:
                 # if there are any data keys not in the safe whitelist
                 output_keys = set(output)
                 if output_keys.difference(safe_keys):
                     return False
         
         return True
     
     def check_cells(self, nb):
         """Return whether all code cells are trusted
         
         If there are no code cells, return True.
         
         This function is the inverse of mark_cells.
         """
         if nb.nbformat < 3:
             return False
         trusted = True
         for cell in yield_code_cells(nb):
             # only distrust a cell if it actually has some output to distrust
             if not self._check_cell(cell, nb.nbformat):
                 trusted = False
 
         return trusted
 
 
 trust_flags = {
     'reset' : (
         {'TrustNotebookApp' : { 'reset' : True}},
         """Generate a new key for notebook signature.
         All previously signed notebooks will become untrusted.
         """
     ),
 }
 trust_flags.update(base_flags)
 trust_flags.pop('init')
 
 
 class TrustNotebookApp(BaseIPythonApplication):
     
     description="""Sign one or more IPython notebooks with your key,
     to trust their dynamic (HTML, Javascript) output.
     
     Trusting a notebook only applies to the current IPython profile.
     To trust a notebook for use with a profile other than default,
     add `--profile [profile name]`.
     
     Otherwise, you will have to re-execute the notebook to see output.
     """
     
     examples = """
     ipython trust mynotebook.ipynb and_this_one.ipynb
     ipython trust --profile myprofile mynotebook.ipynb
     """
     
     flags = trust_flags
     
     reset = Bool(False, config=True,
         help="""If True, generate a new key for notebook signature.
         After reset, all previously signed notebooks will become untrusted.
         """
     )
     
     notary = Instance(NotebookNotary)
     def _notary_default(self):
         return NotebookNotary(parent=self, profile_dir=self.profile_dir)
     
     def sign_notebook(self, notebook_path):
         if not os.path.exists(notebook_path):
             self.log.error("Notebook missing: %s" % notebook_path)
             self.exit(1)
         with io.open(notebook_path, encoding='utf8') as f:
             nb = read(f, NO_CONVERT)
         if self.notary.check_signature(nb):
             print("Notebook already signed: %s" % notebook_path)
         else:
             print("Signing notebook: %s" % notebook_path)
             self.notary.sign(nb)
             with atomic_writing(notebook_path) as f:
                 write(nb, f, NO_CONVERT)
     
     def generate_new_key(self):
         """Generate a new notebook signature key"""
         print("Generating new notebook key: %s" % self.notary.secret_file)
         self.notary._write_secret_file(os.urandom(1024))
     
     def start(self):
         if self.reset:
             self.generate_new_key()
             return
         if not self.extra_args:
             self.log.critical("Specify at least one notebook to sign.")
             self.exit(1)
         
         for notebook_path in self.extra_args:
             self.sign_notebook(notebook_path)
 

IPython/nbformat/tests/test_sign.py

@@ -1,150 +1,200 b''
 """Test Notebook signing"""
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
+import copy
+import time
+
 from .base import TestsBase
 
 from IPython.nbformat import read, sign
 from IPython.core.getipython import get_ipython
 
 
 class TestNotary(TestsBase):
     
     def setUp(self):
         self.notary = sign.NotebookNotary(
             secret=b'secret',
-            profile_dir=get_ipython().profile_dir
+            profile_dir=get_ipython().profile_dir,
+            db_url=':memory:'
         )
         with self.fopen(u'test3.ipynb', u'r') as f:
             self.nb = read(f, as_version=4)
         with self.fopen(u'test3.ipynb', u'r') as f:
             self.nb3 = read(f, as_version=3)
     
     def test_algorithms(self):
         last_sig = ''
         for algo in sign.algorithms:
             self.notary.algorithm = algo
-            self.notary.sign(self.nb)
-            sig = self.nb.metadata.signature
-            print(sig)
-            self.assertEqual(sig[:len(self.notary.algorithm)+1], '%s:' % self.notary.algorithm)
+            sig = self.notary.compute_signature(self.nb)
             self.assertNotEqual(last_sig, sig)
             last_sig = sig
     
     def test_sign_same(self):
         """Multiple signatures of the same notebook are the same"""
         sig1 = self.notary.compute_signature(self.nb)
         sig2 = self.notary.compute_signature(self.nb)
         self.assertEqual(sig1, sig2)
     
     def test_change_secret(self):
         """Changing the secret changes the signature"""
         sig1 = self.notary.compute_signature(self.nb)
         self.notary.secret = b'different'
         sig2 = self.notary.compute_signature(self.nb)
         self.assertNotEqual(sig1, sig2)
     
     def test_sign(self):
+        self.assertFalse(self.notary.check_signature(self.nb))
+        self.notary.sign(self.nb)
+        self.assertTrue(self.notary.check_signature(self.nb))
+    
+    def test_unsign(self):
         self.notary.sign(self.nb)
-        sig = self.nb.metadata.signature
-        self.assertEqual(sig[:len(self.notary.algorithm)+1], '%s:' % self.notary.algorithm)
+        self.assertTrue(self.notary.check_signature(self.nb))
+        self.notary.unsign(self.nb)
+        self.assertFalse(self.notary.check_signature(self.nb))
+        self.notary.unsign(self.nb)
+        self.assertFalse(self.notary.check_signature(self.nb))
+    
+    def test_cull_db(self):
+        # this test has various sleeps of 2ms
+        # to ensure low resolution timestamps compare as expected
+        dt = 2e-3
+        nbs = [
+            copy.deepcopy(self.nb) for i in range(5)
+        ]
+        for i, nb in enumerate(nbs):
+            nb.metadata.dirty = i
+            self.notary.sign(nb)
+        
+        for i, nb in enumerate(nbs):
+            time.sleep(dt)
+            self.assertTrue(self.notary.check_signature(nb), 'nb %i is trusted' % i)
+        
+        self.notary.db_size_limit = 2
+        self.notary.cull_db()
+        
+        # expect all but last two signatures to be culled
+        self.assertEqual(
+            [self.notary.check_signature(nb) for nb in nbs],
+            [False] * (len(nbs) - 2) + [True] * 2
+        )
+        
+        # sign them all again
+        for nb in nbs:
+            time.sleep(dt)
+            self.notary.sign(nb)
+        
+        # checking front two marks them as newest for next cull instead of oldest
+        time.sleep(dt)
+        self.notary.check_signature(nbs[0])
+        self.notary.check_signature(nbs[1])
+        self.notary.cull_db()
+        
+        self.assertEqual(
+            [self.notary.check_signature(nb) for nb in nbs],
+            [True] * 2 + [False] * (len(nbs) - 2)
+        )
+        
     
     def test_check_signature(self):
         nb = self.nb
         md = nb.metadata
         notary = self.notary
         check_signature = notary.check_signature
         # no signature:
         md.pop('signature', None)
         self.assertFalse(check_signature(nb))
         # hash only, no algo
         md.signature = notary.compute_signature(nb)
         self.assertFalse(check_signature(nb))
         # proper signature, algo mismatch
         notary.algorithm = 'sha224'
         notary.sign(nb)
         notary.algorithm = 'sha256'
         self.assertFalse(check_signature(nb))
         # check correctly signed notebook
         notary.sign(nb)
         self.assertTrue(check_signature(nb))
     
     def test_mark_cells_untrusted(self):
         cells = self.nb.cells
         self.notary.mark_cells(self.nb, False)
         for cell in cells:
             self.assertNotIn('trusted', cell)
             if cell.cell_type == 'code':
                 self.assertIn('trusted', cell.metadata)
                 self.assertFalse(cell.metadata.trusted)
             else:
                 self.assertNotIn('trusted', cell.metadata)
     
     def test_mark_cells_trusted(self):
         cells = self.nb.cells
         self.notary.mark_cells(self.nb, True)
         for cell in cells:
             self.assertNotIn('trusted', cell)
             if cell.cell_type == 'code':
                 self.assertIn('trusted', cell.metadata)
                 self.assertTrue(cell.metadata.trusted)
             else:
                 self.assertNotIn('trusted', cell.metadata)
     
     def test_check_cells(self):
         nb = self.nb
         self.notary.mark_cells(nb, True)
         self.assertTrue(self.notary.check_cells(nb))
         for cell in nb.cells:
             self.assertNotIn('trusted', cell)
         self.notary.mark_cells(nb, False)
         self.assertFalse(self.notary.check_cells(nb))
         for cell in nb.cells:
             self.assertNotIn('trusted', cell)
     
     def test_trust_no_output(self):
         nb = self.nb
         self.notary.mark_cells(nb, False)
         for cell in nb.cells:
             if cell.cell_type == 'code':
                 cell.outputs = []
         self.assertTrue(self.notary.check_cells(nb))
     
     def test_mark_cells_untrusted_v3(self):
         nb = self.nb3
         cells = nb.worksheets[0].cells
         self.notary.mark_cells(nb, False)
         for cell in cells:
             self.assertNotIn('trusted', cell)
             if cell.cell_type == 'code':
                 self.assertIn('trusted', cell.metadata)
                 self.assertFalse(cell.metadata.trusted)
             else:
                 self.assertNotIn('trusted', cell.metadata)
     
     def test_mark_cells_trusted_v3(self):
         nb = self.nb3
         cells = nb.worksheets[0].cells
         self.notary.mark_cells(nb, True)
         for cell in cells:
             self.assertNotIn('trusted', cell)
             if cell.cell_type == 'code':
                 self.assertIn('trusted', cell.metadata)
                 self.assertTrue(cell.metadata.trusted)
             else:
                 self.assertNotIn('trusted', cell.metadata)
     
     def test_check_cells_v3(self):
         nb = self.nb3
         cells = nb.worksheets[0].cells
         self.notary.mark_cells(nb, True)
         self.assertTrue(self.notary.check_cells(nb))
         for cell in cells:
             self.assertNotIn('trusted', cell)
         self.notary.mark_cells(nb, False)
         self.assertFalse(self.notary.check_cells(nb))
         for cell in cells:
             self.assertNotIn('trusted', cell)
         
 

IPython/nbformat/v4/convert.py

@@ -1,252 +1,253 b''
 """Code for converting notebooks to and from v3."""
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 import json
 import re
 
 from .nbbase import (
     nbformat, nbformat_minor,
     NotebookNode,
 )
 
 from IPython.nbformat import v3
 from IPython.utils.log import get_logger
 
 def _warn_if_invalid(nb, version):
     """Log validation errors, if there are any."""
     from IPython.nbformat import validate, ValidationError
     try:
         validate(nb, version=version)
     except ValidationError as e:
         get_logger().error("Notebook JSON is not valid v%i: %s", version, e)
 
 def upgrade(nb, from_version=3, from_minor=0):
     """Convert a notebook to v4.
 
     Parameters
     ----------
     nb : NotebookNode
         The Python representation of the notebook to convert.
     from_version : int
         The original version of the notebook to convert.
     from_minor : int
         The original minor version of the notebook to convert (only relevant for v >= 3).
     """
     if from_version == 3:
         # Validate the notebook before conversion
         _warn_if_invalid(nb, from_version)
 
         # Mark the original nbformat so consumers know it has been converted
         orig_nbformat = nb.pop('orig_nbformat', None)
         nb.metadata.orig_nbformat = orig_nbformat or 3
 
         # Mark the new format
         nb.nbformat = nbformat
         nb.nbformat_minor = nbformat_minor
 
         # remove worksheet(s)
         nb['cells'] = cells = []
         # In the unlikely event of multiple worksheets,
         # they will be flattened
         for ws in nb.pop('worksheets', []):
             # upgrade each cell
             for cell in ws['cells']:
                 cells.append(upgrade_cell(cell))
         # upgrade metadata
         nb.metadata.pop('name', '')
+        nb.metadata.pop('signature', '')
         # Validate the converted notebook before returning it
         _warn_if_invalid(nb, nbformat)
         return nb
     elif from_version == 4:
         # nothing to do
         if from_minor != nbformat_minor:
             nb.metadata.orig_nbformat_minor = from_minor
         nb.nbformat_minor = nbformat_minor
 
         return nb
     else:
         raise ValueError('Cannot convert a notebook directly from v%s to v4.  ' \
                 'Try using the IPython.nbformat.convert module.' % from_version)
 
 def upgrade_cell(cell):
     """upgrade a cell from v3 to v4
 
     heading cell:
         - -> markdown heading
     code cell:
         - remove language metadata
         - cell.input -> cell.source
         - cell.prompt_number -> cell.execution_count
         - update outputs
     """
     cell.setdefault('metadata', NotebookNode())
     if cell.cell_type == 'code':
         cell.pop('language', '')
         if 'collapsed' in cell:
             cell.metadata['collapsed'] = cell.pop('collapsed')
         cell.source = cell.pop('input', '')
         cell.execution_count = cell.pop('prompt_number', None)
         cell.outputs = upgrade_outputs(cell.outputs)
     elif cell.cell_type == 'heading':
         cell.cell_type = 'markdown'
         level = cell.pop('level', 1)
         cell.source = u'{hashes} {single_line}'.format(
             hashes='#' * level,
             single_line = ' '.join(cell.get('source', '').splitlines()),
         )
     elif cell.cell_type == 'html':
         # Technically, this exists. It will never happen in practice.
         cell.cell_type = 'markdown'
     return cell
 
 def downgrade_cell(cell):
     """downgrade a cell from v4 to v3
 
     code cell:
         - set cell.language
         - cell.input <- cell.source
         - cell.prompt_number <- cell.execution_count
         - update outputs
     markdown cell:
         - single-line heading -> heading cell
     """
     if cell.cell_type == 'code':
         cell.language = 'python'
         cell.input = cell.pop('source', '')
         cell.prompt_number = cell.pop('execution_count', None)
         cell.collapsed = cell.metadata.pop('collapsed', False)
         cell.outputs = downgrade_outputs(cell.outputs)
     elif cell.cell_type == 'markdown':
         source = cell.get('source', '')
         if '\n' not in source and source.startswith('#'):
             prefix, text = re.match(r'(#+)\s*(.*)', source).groups()
             cell.cell_type = 'heading'
             cell.source = text
             cell.level = len(prefix)
     return cell
 
 _mime_map = {
     "text" : "text/plain",
     "html" : "text/html",
     "svg" : "image/svg+xml",
     "png" : "image/png",
     "jpeg" : "image/jpeg",
     "latex" : "text/latex",
     "json" : "application/json",
     "javascript" : "application/javascript",
 };
 
 def to_mime_key(d):
     """convert dict with v3 aliases to plain mime-type keys"""
     for alias, mime in _mime_map.items():
         if alias in d:
             d[mime] = d.pop(alias)
     return d
 
 def from_mime_key(d):
     """convert dict with mime-type keys to v3 aliases"""
     for alias, mime in _mime_map.items():
         if mime in d:
             d[alias] = d.pop(mime)
     return d
 
 def upgrade_output(output):
     """upgrade a single code cell output from v3 to v4
 
     - pyout -> execute_result
     - pyerr -> error
     - output.type -> output.data.mime/type
     - mime-type keys
     - stream.stream -> stream.name
     """
     if output['output_type'] in {'pyout', 'display_data'}:
         output.setdefault('metadata', NotebookNode())
         if output['output_type'] == 'pyout':
             output['output_type'] = 'execute_result'
             output['execution_count'] = output.pop('prompt_number', None)
 
         # move output data into data sub-dict
         data = {}
         for key in list(output):
             if key in {'output_type', 'execution_count', 'metadata'}:
                 continue
             data[key] = output.pop(key)
         to_mime_key(data)
         output['data'] = data
         to_mime_key(output.metadata)
         if 'application/json' in data:
             data['application/json'] = json.loads(data['application/json'])
         # promote ascii bytes (from v2) to unicode
         for key in ('image/png', 'image/jpeg'):
             if key in data and isinstance(data[key], bytes):
                 data[key] = data[key].decode('ascii')
     elif output['output_type'] == 'pyerr':
         output['output_type'] = 'error'
     elif output['output_type'] == 'stream':
         output['name'] = output.pop('stream')
     return output
 
 def downgrade_output(output):
     """downgrade a single code cell output to v3 from v4
 
     - pyout <- execute_result
     - pyerr <- error
     - output.data.mime/type -> output.type
     - un-mime-type keys
     - stream.stream <- stream.name
     """
     if output['output_type'] in {'execute_result', 'display_data'}:
         if output['output_type'] == 'execute_result':
             output['output_type'] = 'pyout'
             output['prompt_number'] = output.pop('execution_count', None)
 
         # promote data dict to top-level output namespace
         data = output.pop('data', {})
         if 'application/json' in data:
             data['application/json'] = json.dumps(data['application/json'])
         from_mime_key(data)
         output.update(data)
         from_mime_key(output.get('metadata', {}))
     elif output['output_type'] == 'error':
         output['output_type'] = 'pyerr'
     elif output['output_type'] == 'stream':
         output['stream'] = output.pop('name')
     return output
 
 def upgrade_outputs(outputs):
     """upgrade outputs of a code cell from v3 to v4"""
     return [upgrade_output(op) for op in outputs]
 
 def downgrade_outputs(outputs):
     """downgrade outputs of a code cell to v3 from v4"""
     return [downgrade_output(op) for op in outputs]
 
 def downgrade(nb):
     """Convert a v4 notebook to v3.
 
     Parameters
     ----------
     nb : NotebookNode
         The Python representation of the notebook to convert.
     """
     if nb.nbformat != nbformat:
         return nb
 
     # Validate the notebook before conversion
     _warn_if_invalid(nb, nbformat)
 
     nb.nbformat = v3.nbformat
     nb.nbformat_minor = v3.nbformat_minor
     cells = [ downgrade_cell(cell) for cell in nb.pop('cells') ]
     nb.worksheets = [v3.new_worksheet(cells=cells)]
     nb.metadata.setdefault('name', '')
     
     # Validate the converted notebook before returning it
     _warn_if_invalid(nb, v3.nbformat)
     
     nb.orig_nbformat = nb.metadata.pop('orig_nbformat', nbformat)
     nb.orig_nbformat_minor = nb.metadata.pop('orig_nbformat_minor', nbformat_minor)
     
     return nb

IPython/nbformat/v4/nbformat.v4.schema.json

@@ -1,375 +1,371 b''
 {
     "$schema": "http://json-schema.org/draft-04/schema#",
     "description": "IPython Notebook v4.0 JSON schema.",
     "type": "object",
     "additionalProperties": false,
     "required": ["metadata", "nbformat_minor", "nbformat", "cells"],
     "properties": {
         "metadata": {
             "description": "Notebook root-level metadata.",
             "type": "object",
             "additionalProperties": true,
             "properties": {
                 "kernelspec": {
                     "description": "Kernel information.",
                     "type": "object",
                     "required": ["name", "display_name"],
                     "properties": {
                         "name": {
                             "description": "Name of the kernel specification.",
                             "type": "string"
                         },
                         "display_name": {
                             "description": "Name to display in UI.",
                             "type": "string"
                         }
                     }
                 },
                 "language_info": {
                   "description": "Kernel information.",
                   "type": "object",
                   "required": ["name"],
                   "properties": {
                     "name": {
                         "description": "The programming language which this kernel runs.",
                         "type": "string"
                     },
                     "codemirror_mode": {
                         "description": "The codemirror mode to use for code in this language.",
                         "oneOf": [
                           {"type": "string"},
                           {"type": "object"}
                         ]
                     },
                     "file_extension": {
                         "description": "The file extension for files in this language.",
                         "type": "string"
                     },
                     "mimetype": {
                         "description": "The mimetype corresponding to files in this language.",
                         "type": "string"
                     },
                     "pygments_lexer": {
                         "description": "The pygments lexer to use for code in this language.",
                         "type": "string"
                     }
                   }
                 },
-                "signature": {
-                    "description": "Hash of the notebook.",
-                    "type": "string"
-                },
                 "orig_nbformat": {
                     "description": "Original notebook format (major number) before converting the notebook between versions. This should never be written to a file.",
                     "type": "integer",
                     "minimum": 1
                 }
             }
         },
         "nbformat_minor": {
             "description": "Notebook format (minor number). Incremented for backward compatible changes to the notebook format.",
             "type": "integer",
             "minimum": 0
         },
         "nbformat": {
             "description": "Notebook format (major number). Incremented between backwards incompatible changes to the notebook format.",
             "type": "integer",
             "minimum": 4,
             "maximum": 4
         },
         "cells": {
             "description": "Array of cells of the current notebook.",
             "type": "array",
             "items": {"$ref": "#/definitions/cell"}
         }
     },
 
     "definitions": {
         "cell": {
             "type": "object",
             "oneOf": [
                 {"$ref": "#/definitions/raw_cell"},
                 {"$ref": "#/definitions/markdown_cell"},
                 {"$ref": "#/definitions/code_cell"}
             ]
         },
 
         "raw_cell": {
             "description": "Notebook raw nbconvert cell.",
             "type": "object",
             "additionalProperties": false,
             "required": ["cell_type", "metadata", "source"],
             "properties": {
                 "cell_type": {
                     "description": "String identifying the type of cell.",
                     "enum": ["raw"]
                 },
                 "metadata": {
                     "description": "Cell-level metadata.",
                     "type": "object",
                     "additionalProperties": true,
                     "properties": {
                         "format": {
                             "description": "Raw cell metadata format for nbconvert.",
                             "type": "string"
                         },
                         "name": {"$ref": "#/definitions/misc/metadata_name"},
                         "tags": {"$ref": "#/definitions/misc/metadata_tags"}
                     }
                 },
                 "source": {"$ref": "#/definitions/misc/source"}
             }
         },
 
         "markdown_cell": {
             "description": "Notebook markdown cell.",
             "type": "object",
             "additionalProperties": false,
             "required": ["cell_type", "metadata", "source"],
             "properties": {
                 "cell_type": {
                     "description": "String identifying the type of cell.",
                     "enum": ["markdown"]
                 },
                 "metadata": {
                     "description": "Cell-level metadata.",
                     "type": "object",
                     "properties": {
                         "name": {"$ref": "#/definitions/misc/metadata_name"},
                         "tags": {"$ref": "#/definitions/misc/metadata_tags"}
                     },
                     "additionalProperties": true
                 },
                 "source": {"$ref": "#/definitions/misc/source"}
             }
         },
 
         "code_cell": {
             "description": "Notebook code cell.",
             "type": "object",
             "additionalProperties": false,
             "required": ["cell_type", "metadata", "source", "outputs", "execution_count"],
             "properties": {
                 "cell_type": {
                     "description": "String identifying the type of cell.",
                     "enum": ["code"]
                 },
                 "metadata": {
                     "description": "Cell-level metadata.",
                     "type": "object",
                     "additionalProperties": true,
                     "properties": {
                         "collapsed": {
                             "description": "Whether the cell is collapsed/expanded.",
                             "type": "boolean"
                         },
                         "autoscroll": {
                             "description": "Whether the cell's output is scrolled, unscrolled, or autoscrolled.",
                             "enum": [true, false, "auto"]
                         },
                         "name": {"$ref": "#/definitions/misc/metadata_name"},
                         "tags": {"$ref": "#/definitions/misc/metadata_tags"}
                     }
                 },
                 "source": {"$ref": "#/definitions/misc/source"},
                 "outputs": {
                     "description": "Execution, display, or stream outputs.",
                     "type": "array",
                     "items": {"$ref": "#/definitions/output"}
                 },
                 "execution_count": {
                     "description": "The code cell's prompt number. Will be null if the cell has not been run.",
                     "type": ["integer", "null"],
                     "minimum": 0
                 }
             }
         },
         
         "unrecognized_cell": {
             "description": "Unrecognized cell from a future minor-revision to the notebook format.",
             "type": "object",
             "additionalProperties": true,
             "required": ["cell_type", "metadata"],
             "properties": {
                 "cell_type": {
                     "description": "String identifying the type of cell.",
                     "not" : {
                       "enum": ["markdown", "code", "raw"]
                     }
                 },
                 "metadata": {
                     "description": "Cell-level metadata.",
                     "type": "object",
                     "properties": {
                         "name": {"$ref": "#/definitions/misc/metadata_name"},
                         "tags": {"$ref": "#/definitions/misc/metadata_tags"}
                     },
                     "additionalProperties": true
                 }
             }
         },
         
         "output": {
             "type": "object",
             "oneOf": [
                 {"$ref": "#/definitions/execute_result"},
                 {"$ref": "#/definitions/display_data"},
                 {"$ref": "#/definitions/stream"},
                 {"$ref": "#/definitions/error"}
             ]
         },
 
         "execute_result": {
             "description": "Result of executing a code cell.",
             "type": "object",
             "additionalProperties": false,
             "required": ["output_type", "data", "metadata", "execution_count"],
             "properties": {
                 "output_type": {
                     "description": "Type of cell output.",
                     "enum": ["execute_result"]
                 },
                 "execution_count": {
                     "description": "A result's prompt number.",
                     "type": ["integer", "null"],
                     "minimum": 0
                 },
                 "data": {"$ref": "#/definitions/misc/mimebundle"},
                 "metadata": {"$ref": "#/definitions/misc/output_metadata"}
             }
         },
 
         "display_data": {
             "description": "Data displayed as a result of code cell execution.",
             "type": "object",
             "additionalProperties": false,
             "required": ["output_type", "data", "metadata"],
             "properties": {
                 "output_type": {
                     "description": "Type of cell output.",
                     "enum": ["display_data"]
                 },
                 "data": {"$ref": "#/definitions/misc/mimebundle"},
                 "metadata": {"$ref": "#/definitions/misc/output_metadata"}
             }
         },
 
         "stream": {
             "description": "Stream output from a code cell.",
             "type": "object",
             "additionalProperties": false,
             "required": ["output_type", "name", "text"],
             "properties": {
                 "output_type": {
                     "description": "Type of cell output.",
                     "enum": ["stream"]
                 },
                 "name": {
                     "description": "The name of the stream (stdout, stderr).",
                     "type": "string"
                 },
                 "text": {
                     "description": "The stream's text output, represented as an array of strings.",
                     "$ref": "#/definitions/misc/multiline_string"
                 }
             }
         },
 
         "error": {
             "description": "Output of an error that occurred during code cell execution.",
             "type": "object",
             "additionalProperties": false,
             "required": ["output_type", "ename", "evalue", "traceback"],
             "properties": {
                 "output_type": {
                     "description": "Type of cell output.",
                     "enum": ["error"]
                 },
                 "ename": {
                     "description": "The name of the error.",
                     "type": "string"
                 },
                 "evalue": {
                     "description": "The value, or message, of the error.",
                     "type": "string"
                 },
                 "traceback": {
                     "description": "The error's traceback, represented as an array of strings.",
                     "type": "array",
                     "items": {"type": "string"}
                 }
             }
         },
 
         "unrecognized_output": {
             "description": "Unrecognized output from a future minor-revision to the notebook format.",
             "type": "object",
             "additionalProperties": true,
             "required": ["output_type"],
             "properties": {
                 "output_type": {
                     "description": "Type of cell output.",
                     "not": {
                         "enum": ["execute_result", "display_data", "stream", "error"]
                     }
                 }
             }
         },
 
         "misc": {
             "metadata_name": {
                 "description": "The cell's name. If present, must be a non-empty string.",
                 "type": "string",
                 "pattern": "^.+$"
             },
             "metadata_tags": {
                 "description": "The cell's tags. Tags must be unique, and must not contain commas.",
                 "type": "array",
                 "uniqueItems": true,
                 "items": {
                     "type": "string",
                     "pattern": "^[^,]+$"
                 }
             },
             "source": {
                 "description": "Contents of the cell, represented as an array of lines.",
                 "$ref": "#/definitions/misc/multiline_string"
             },
             "execution_count": {
                 "description": "The code cell's prompt number. Will be null if the cell has not been run.",
                 "type": ["integer", "null"],
                 "minimum": 0
             },
             "mimebundle": {
                 "description": "A mime-type keyed dictionary of data",
                 "type": "object",
                 "additionalProperties": false,
                 "properties": {
                     "application/json": {
                         "type": "object"
                     }
                 },
                 "patternProperties": {
                     "^(?!application/json$)[a-zA-Z0-9]+/[a-zA-Z0-9\\-\\+\\.]+$": {
                         "description": "mimetype output (e.g. text/plain), represented as either an array of strings or a string.",
                         "$ref": "#/definitions/misc/multiline_string"
                     }
                 }
             },
             "output_metadata": {
                 "description": "Cell output metadata.",
                 "type": "object",
                 "additionalProperties": true
             },
             "multiline_string": {
                 "oneOf" : [
                     {"type": "string"},
                     {
                         "type": "array",
                         "items": {"type": "string"}
                     }
                 ]
             }
         }
     }
 }

IPython/nbformat/v4/rwbase.py

@@ -1,95 +1,96 b''
 """Base classes and utilities for readers and writers."""
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 from IPython.utils.py3compat import string_types, cast_unicode_py2
 
 
 def rejoin_lines(nb):
     """rejoin multiline text into strings
 
     For reversing effects of ``split_lines(nb)``.
 
     This only rejoins lines that have been split, so if text objects were not split
     they will pass through unchanged.
 
     Used when reading JSON files that may have been passed through split_lines.
     """
     for cell in nb.cells:
         if 'source' in cell and isinstance(cell.source, list):
             cell.source = ''.join(cell.source)
         if cell.get('cell_type', None) == 'code':
             for output in cell.get('outputs', []):
                 output_type = output.get('output_type', '')
                 if output_type in {'execute_result', 'display_data'}:
                     for key, value in output.get('data', {}).items():
                         if key != 'application/json' and isinstance(value, list):
                             output.data[key] = ''.join(value)
                 elif output_type:
                     if isinstance(output.get('text', ''), list):
                         output.text = ''.join(output.text)
     return nb
 
 
 def split_lines(nb):
     """split likely multiline text into lists of strings
 
     For file output more friendly to line-based VCS. ``rejoin_lines(nb)`` will
     reverse the effects of ``split_lines(nb)``.
 
     Used when writing JSON files.
     """
     for cell in nb.cells:
         source = cell.get('source', None)
         if isinstance(source, string_types):
             cell['source'] = source.splitlines(True)
 
         if cell.cell_type == 'code':
             for output in cell.outputs:
                 if output.output_type in {'execute_result', 'display_data'}:
                     for key, value in output.data.items():
                         if key != 'application/json' and isinstance(value, string_types):
                             output.data[key] = value.splitlines(True)
                 elif output.output_type == 'stream':
                     if isinstance(output.text, string_types):
                         output.text = output.text.splitlines(True)
     return nb
 
 
 def strip_transient(nb):
     """Strip transient values that shouldn't be stored in files.
 
     This should be called in *both* read and write.
     """
     nb.metadata.pop('orig_nbformat', None)
     nb.metadata.pop('orig_nbformat_minor', None)
+    nb.metadata.pop('signature', None)
     for cell in nb.cells:
         cell.metadata.pop('trusted', None)
     return nb
 
 
 class NotebookReader(object):
     """A class for reading notebooks."""
 
     def reads(self, s, **kwargs):
         """Read a notebook from a string."""
         raise NotImplementedError("loads must be implemented in a subclass")
 
     def read(self, fp, **kwargs):
         """Read a notebook from a file like object"""
         nbs = cast_unicode_py2(fp.read())
         return self.reads(nbs, **kwargs)
 
 
 class NotebookWriter(object):
     """A class for writing notebooks."""
 
     def writes(self, nb, **kwargs):
         """Write a notebook to a string."""
         raise NotImplementedError("loads must be implemented in a subclass")
 
     def write(self, nb, fp, **kwargs):
         """Write a notebook to a file like object"""
         nbs = cast_unicode_py2(self.writes(nb, **kwargs))
         return fp.write(nbs)