strip transient values to/from nb files

orig_nbformat isn’t part of the file format

IPython/nbformat/v4/nbformat.v4.schema.json

@@ -1,346 +1,346 b''
 {
     "$schema": "http://json-schema.org/draft-04/schema#",
     "description": "IPython Notebook v4.0 JSON schema.",
     "type": "object",
     "additionalProperties": false,
     "required": ["metadata", "nbformat_minor", "nbformat", "cells"],
     "properties": {
         "metadata": {
             "description": "Notebook root-level metadata.",
             "type": "object",
             "additionalProperties": true,
             "properties": {
                 "kernel_info": {
                     "description": "Kernel information.",
                     "type": "object",
                     "required": ["name", "language"],
                     "properties": {
                         "name": {
                             "description": "Name of the kernel specification.",
                             "type": "string"
                         },
                         "language": {
                             "description": "The programming language which this kernel runs.",
                             "type": "string"
                         },
                         "codemirror_mode": {
                             "description": "The codemirror mode to use for code in this language.",
                             "type": "string"
                         }
                     }
                 },
                 "signature": {
                     "description": "Hash of the notebook.",
                     "type": "string"
                 },
                 "orig_nbformat": {
-                    "description": "Original notebook format (major number) before converting the notebook between versions.",
+                    "description": "Original notebook format (major number) before converting the notebook between versions. This should never be written to a file.",
                     "type": "integer",
                     "minimum": 1
                 }
             }
         },
         "nbformat_minor": {
             "description": "Notebook format (minor number). Incremented for backward compatible changes to the notebook format.",
             "type": "integer",
             "minimum": 0
         },
         "nbformat": {
             "description": "Notebook format (major number). Incremented between backwards incompatible changes to the notebook format.",
             "type": "integer",
             "minimum": 4,
             "maximum": 4
         },
         "cells": {
             "description": "Array of cells of the current notebook.",
             "type": "array",
             "items": {
                 "type": "object",
                 "oneOf": [
                     {"$ref": "#/definitions/raw_cell"},
                     {"$ref": "#/definitions/markdown_cell"},
                     {"$ref": "#/definitions/heading_cell"},
                     {"$ref": "#/definitions/code_cell"}
                 ]
             }
         }
     },
 
     "definitions": {
 
         "raw_cell": {
             "description": "Notebook raw nbconvert cell.",
             "type": "object",
             "additionalProperties": false,
             "required": ["cell_type", "metadata", "source"],
             "properties": {
                 "cell_type": {
                     "description": "String identifying the type of cell.",
                     "enum": ["raw"]
                 },
                 "metadata": {
                     "description": "Cell-level metadata.",
                     "type": "object",
                     "additionalProperties": true,
                     "properties": {
                         "format": {
                             "description": "Raw cell metadata format for nbconvert.",
                             "type": "string"
                         },
                         "name": {"$ref": "#/definitions/misc/metadata_name"},
                         "tags": {"$ref": "#/definitions/misc/metadata_tags"}
                     }
                 },
                 "source": {"$ref": "#/definitions/misc/source"}
             }
         },
 
         "markdown_cell": {
             "description": "Notebook markdown cell.",
             "type": "object",
             "additionalProperties": false,
             "required": ["cell_type", "metadata", "source"],
             "properties": {
                 "cell_type": {
                     "description": "String identifying the type of cell.",
                     "enum": ["markdown"]
                 },
                 "metadata": {
                     "description": "Cell-level metadata.",
                     "type": "object",
                     "properties": {
                         "name": {"$ref": "#/definitions/misc/metadata_name"},
                         "tags": {"$ref": "#/definitions/misc/metadata_tags"}
                     },
                     "additionalProperties": true
                 },
                 "source": {"$ref": "#/definitions/misc/source"}
             }
         },
 
         "heading_cell": {
             "description": "Notebook heading cell.",
             "type": "object",
             "additionalProperties": false,
             "required": ["cell_type", "metadata", "source", "level"],
             "properties": {
                 "cell_type": {
                     "description": "String identifying the type of cell.",
                     "enum": ["heading"]
                 },
                 "metadata": {
                     "description": "Cell-level metadata.",
                     "type": "object",
                     "properties": {
                         "name": {"$ref": "#/definitions/misc/metadata_name"},
                         "tags": {"$ref": "#/definitions/misc/metadata_tags"}
                     },
                     "additionalProperties": true
                 },
                 "source": {"$ref": "#/definitions/misc/source"},
                 "level": {
                     "description": "Level of heading cells.",
                     "type": "integer",
                     "minimum": 1
                 }
             }
         },
 
         "code_cell": {
             "description": "Notebook code cell.",
             "type": "object",
             "additionalProperties": false,
             "required": ["cell_type", "metadata", "source", "outputs", "prompt_number"],
             "properties": {
                 "cell_type": {
                     "description": "String identifying the type of cell.",
                     "enum": ["code"]
                 },
                 "metadata": {
                     "description": "Cell-level metadata.",
                     "type": "object",
                     "additionalProperties": true,
                     "properties": {
                         "collapsed": {
                             "description": "Whether the cell is collapsed/expanded.",
                             "type": "boolean"
                         },
                         "autoscroll": {
                             "description": "Whether the cell's output is scrolled, unscrolled, or autoscrolled.",
                             "enum": [true, false, "auto"]
                         },
                         "name": {"$ref": "#/definitions/misc/metadata_name"},
                         "tags": {"$ref": "#/definitions/misc/metadata_tags"}
                     }
                 },
                 "source": {"$ref": "#/definitions/misc/source"},
                 "outputs": {
                     "description": "Execution, display, or stream outputs.",
                     "type": "array",
                     "items": {"$ref": "#/definitions/output"}
                 },
                 "prompt_number": {
                     "description": "The code cell's prompt number. Will be null if the cell has not been run.",
                     "type": ["integer", "null"],
                     "minimum": 0
                 }
             }
         },
         "output": {
             "type": "object",
             "oneOf": [
                 {"$ref": "#/definitions/execute_result"},
                 {"$ref": "#/definitions/display_data"},
                 {"$ref": "#/definitions/stream"},
                 {"$ref": "#/definitions/error"}
             ]
         },
         "execute_result": {
             "description": "Result of executing a code cell.",
             "type": "object",
             "additionalProperties": false,
             "required": ["output_type", "metadata", "prompt_number"],
             "properties": {
                 "output_type": {
                     "description": "Type of cell output.",
                     "enum": ["execute_result"]
                 },
                 "prompt_number": {
                     "description": "A result's prompt number.",
                     "type": ["integer"],
                     "minimum": 0
                 },
                 "application/json": {
                     "type": "object"
                 },
                 "metadata": {"$ref": "#/definitions/misc/output_metadata"}
             },
             "patternProperties": {
                 "^(?!application/json$)[a-zA-Z0-9]+/[a-zA-Z0-9\\-\\+\\.]+$": {
                     "description": "mimetype output (e.g. text/plain), represented as either an array of strings or a string.",
                     "$ref": "#/definitions/misc/multiline_string"
                 }
             }
         },
 
         "display_data": {
             "description": "Data displayed as a result of code cell execution.",
             "type": "object",
             "additionalProperties": false,
             "required": ["output_type", "metadata"],
             "properties": {
                 "output_type": {
                     "description": "Type of cell output.",
                     "enum": ["display_data"]
                 },
                 "application/json": {
                     "type": "object"
                 },
                 "metadata": {"$ref": "#/definitions/misc/output_metadata"}
             },
             "patternProperties": {
                 "^(?!application/json$)[a-zA-Z0-9]+/[a-zA-Z0-9\\-\\+\\.]+$": {
                     "description": "mimetype output (e.g. text/plain), represented as either an array of strings or a string.",
                     "$ref": "#/definitions/misc/multiline_string"
                 }
             }
         },
 
         "stream": {
             "description": "Stream output from a code cell.",
             "type": "object",
             "additionalProperties": false,
             "required": ["output_type", "metadata", "name", "text"],
             "properties": {
                 "output_type": {
                     "description": "Type of cell output.",
                     "enum": ["stream"]
                 },
                 "metadata": {"$ref": "#/definitions/misc/output_metadata"},
                 "name": {
                     "description": "The name of the stream (stdout, stderr).",
                     "type": "string"
                 },
                 "text": {
                     "description": "The stream's text output, represented as an array of strings.",
                     "$ref": "#/definitions/misc/multiline_string"
                 }
             }
         },
 
         "error": {
             "description": "Output of an error that occurred during code cell execution.",
             "type": "object",
             "additionalProperties": false,
             "required": ["output_type", "metadata", "ename", "evalue", "traceback"],
             "properties": {
                 "output_type": {
                     "description": "Type of cell output.",
                     "enum": ["error"]
                 },
                 "metadata": {"$ref": "#/definitions/misc/output_metadata"},
                 "ename": {
                     "description": "The name of the error.",
                     "type": "string"
                 },
                 "evalue": {
                     "description": "The value, or message, of the error.",
                     "type": "string"
                 },
                 "traceback": {
                     "description": "The error's traceback, represented as an array of strings.",
                     "type": "array",
                     "items": {"type": "string"}
                 }
             }
         },
 
         "misc": {
             "metadata_name": {
                 "description": "The cell's name. If present, must be a non-empty string.",
                 "type": "string",
                 "pattern": "^.+$"
             },
             "metadata_tags": {
                 "description": "The cell's tags. Tags must be unique, and must not contain commas.",
                 "type": "array",
                 "uniqueItems": true,
                 "items": {
                     "type": "string",
                     "pattern": "^[^,]+$"
                 }
             },
             "source": {
                 "description": "Contents of the cell, represented as an array of lines.",
                 "$ref": "#/definitions/misc/multiline_string"
             },
             "prompt_number": {
                 "description": "The code cell's prompt number. Will be null if the cell has not been run.",
                 "type": ["integer", "null"],
                 "minimum": 0
             },
             "mimetype": {
                 "patternProperties": {
                     "^[a-zA-Z0-9\\-\\+]+/[a-zA-Z0-9\\-\\+]+": {
                         "description": "The cell's mimetype output (e.g. text/plain), represented as either an array of strings or a string.",
                         "$ref": "#/definitions/misc/multiline_string"
                     }
                 }
             },
             "output_metadata": {
                 "description": "Cell output metadata.",
                 "type": "object",
                 "additionalProperties": true
             },
             "multiline_string": {
                 "oneOf" : [
                     {"type": "string"},
                     {
                         "type": "array",
                         "items": {"type": "string"}
                     }
                 ]
             }
         }
     }
 }

IPython/nbformat/v4/nbjson.py

@@ -1,55 +1,60 b''
 """Read and write notebooks in JSON format."""
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 import copy
 import json
 
 from IPython.utils import py3compat
 
 from .nbbase import from_dict
 from .rwbase import (
-    NotebookReader, NotebookWriter, rejoin_lines, split_lines
+    NotebookReader, NotebookWriter, rejoin_lines, split_lines, strip_transient
 )
 
 
 class BytesEncoder(json.JSONEncoder):
     """A JSON encoder that accepts b64 (and other *ascii*) bytestrings."""
     def default(self, obj):
         if isinstance(obj, bytes):
             return obj.decode('ascii')
         return json.JSONEncoder.default(self, obj)
 
 
 class JSONReader(NotebookReader):
 
     def reads(self, s, **kwargs):
         nb = json.loads(s, **kwargs)
         nb = self.to_notebook(nb, **kwargs)
         return nb
 
     def to_notebook(self, d, **kwargs):
-        return rejoin_lines(from_dict(d))
+        nb = rejoin_lines(from_dict(d))
+        nb = strip_transient(nb)
+        return nb
 
 
 class JSONWriter(NotebookWriter):
 
     def writes(self, nb, **kwargs):
         kwargs['cls'] = BytesEncoder
         kwargs['indent'] = 1
         kwargs['sort_keys'] = True
         kwargs['separators'] = (',',': ')
+        # don't modify in-memory dict
+        nb = copy.deepcopy(nb)
         if kwargs.pop('split_lines', True):
-            nb = split_lines(copy.deepcopy(nb))
+            nb = split_lines(nb)
+        nb = strip_transient(nb)
         return py3compat.str_to_unicode(json.dumps(nb, **kwargs), 'utf-8')
 
 
 _reader = JSONReader()
 _writer = JSONWriter()
 
 reads = _reader.reads
 read = _reader.read
 to_notebook = _reader.to_notebook
 write = _writer.write
 writes = _writer.writes

IPython/nbformat/v4/rwbase.py

@@ -1,106 +1,118 b''
 """Base classes and utilities for readers and writers."""
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 from IPython.utils import py3compat
 from IPython.utils.py3compat import unicode_type, string_types
 
 # output keys that are likely to have multiline values
 _multiline_outputs = {
     'text/plain',
     'text/html',
     'image/svg+xml',
     'text/latex',
     'application/javascript',
 }
 
 
 # FIXME: workaround for old splitlines()
 def _join_lines(lines):
     """join lines that have been written by splitlines()
 
     Has logic to protect against `splitlines()`, which
     should have been `splitlines(True)`
     """
     if lines and lines[0].endswith(('\n', '\r')):
         # created by splitlines(True)
         return u''.join(lines)
     else:
         # created by splitlines()
         return u'\n'.join(lines)
 
 
 def rejoin_lines(nb):
     """rejoin multiline text into strings
 
     For reversing effects of ``split_lines(nb)``.
 
     This only rejoins lines that have been split, so if text objects were not split
     they will pass through unchanged.
 
     Used when reading JSON files that may have been passed through split_lines.
     """
     for cell in nb.cells:
         if 'source' in cell and isinstance(cell.source, list):
             cell.source = _join_lines(cell.source)
         if cell.cell_type == 'code':
             for output in cell.outputs:
                 for key in _multiline_outputs:
                     item = output.get(key, None)
                     if isinstance(item, list):
                         output[key] = _join_lines(item)
     return nb
 
 
 def split_lines(nb):
     """split likely multiline text into lists of strings
 
     For file output more friendly to line-based VCS. ``rejoin_lines(nb)`` will
     reverse the effects of ``split_lines(nb)``.
 
     Used when writing JSON files.
     """
     for cell in nb.cells:
         source = cell.get('source', None)
         if isinstance(source, string_types):
             cell['source'] = source.splitlines(True)
 
         if cell.cell_type == 'code':
             for output in cell.outputs:
                 for key in _multiline_outputs:
                     item = output.get(key, None)
                     if isinstance(item, string_types):
                         output[key] = item.splitlines(True)
     return nb
 
 
+def strip_transient(nb):
+    """Strip transient values that shouldn't be stored in files.
+
+    This should be called in *both* read and write.
+    """
+    nb.metadata.pop('orig_nbformat', None)
+    nb.metadata.pop('orig_nbformat_minor', None)
+    for cell in nb.cells:
+        cell.metadata.pop('trusted', None)
+    return nb
+
+
 class NotebookReader(object):
     """A class for reading notebooks."""
 
     def reads(self, s, **kwargs):
         """Read a notebook from a string."""
         raise NotImplementedError("loads must be implemented in a subclass")
 
     def read(self, fp, **kwargs):
         """Read a notebook from a file like object"""
         nbs = fp.read()
         if not py3compat.PY3 and not isinstance(nbs, unicode_type):
             nbs = py3compat.str_to_unicode(nbs)
         return self.reads(nbs, **kwargs)
 
 
 class NotebookWriter(object):
     """A class for writing notebooks."""
 
     def writes(self, nb, **kwargs):
         """Write a notebook to a string."""
         raise NotImplementedError("loads must be implemented in a subclass")
 
     def write(self, nb, fp, **kwargs):
         """Write a notebook to a file like object"""
         nbs = self.writes(nb,**kwargs)
         if not py3compat.PY3 and not isinstance(nbs, unicode_type):
             # this branch is likely only taken for JSON on Python 2
             nbs = py3compat.str_to_unicode(nbs)
         return fp.write(nbs)