From e31440def2decf0bf687d7ca8dd649685791980b 2014-10-08 19:33:31 From: MinRK Date: 2014-10-08 19:33:31 Subject: [PATCH] Use Draft4 JSON Schema for both v3 and v4 no longer need jsonpointer --- diff --git a/IPython/nbformat/current.py b/IPython/nbformat/current.py index aa00cb4..c3ef31a 100644 --- a/IPython/nbformat/current.py +++ b/IPython/nbformat/current.py @@ -1,5 +1,8 @@ """The official API for working with notebooks in the current format version.""" +# Copyright (c) IPython Development Team. +# Distributed under the terms of the Modified BSD License. + from __future__ import print_function import re @@ -17,7 +20,7 @@ from IPython.nbformat import v3 as _v_latest from .reader import reads as reader_reads from .reader import versions from .convert import convert -from .validator import validate +from .validator import validate, ValidationError from IPython.utils.log import get_logger @@ -60,11 +63,10 @@ def reads_json(nbjson, **kwargs): """ nb = reader_reads(nbjson, **kwargs) nb_current = convert(nb, current_nbformat) - errors = validate(nb_current) - if errors: - get_logger().error( - "Notebook JSON is invalid (%d errors detected during read)", - len(errors)) + try: + validate(nb_current) + except ValidationError as e: + get_logger().error("Notebook JSON is invalid: %s", e) return nb_current @@ -73,11 +75,10 @@ def writes_json(nb, **kwargs): any JSON format errors are detected. """ - errors = validate(nb) - if errors: - get_logger().error( - "Notebook JSON is invalid (%d errors detected during write)", - len(errors)) + try: + validate(nb) + except ValidationError as e: + get_logger().error("Notebook JSON is invalid: %s", e) nbjson = versions[current_nbformat].writes_json(nb, **kwargs) return nbjson diff --git a/IPython/nbformat/tests/test_validator.py b/IPython/nbformat/tests/test_validator.py index 9706a96..2778105 100644 --- a/IPython/nbformat/tests/test_validator.py +++ b/IPython/nbformat/tests/test_validator.py @@ -1,23 +1,14 @@ -""" -Contains tests class for validator.py -""" -#----------------------------------------------------------------------------- -# Copyright (C) 2014 The IPython Development Team -# -# Distributed under the terms of the BSD License. The full license is in -# the file COPYING, distributed as part of this software. -#----------------------------------------------------------------------------- +"""Test nbformat.validator""" -#----------------------------------------------------------------------------- -# Imports -#----------------------------------------------------------------------------- +# Copyright (c) IPython Development Team. +# Distributed under the terms of the Modified BSD License. import os from .base import TestsBase -from jsonschema import SchemaError +from jsonschema import ValidationError from ..current import read -from ..validator import schema_path, isvalid, validate, resolve_ref +from ..validator import isvalid, validate #----------------------------------------------------------------------------- @@ -26,22 +17,18 @@ from ..validator import schema_path, isvalid, validate, resolve_ref class TestValidator(TestsBase): - def test_schema_path(self): - """Test that the schema path exists""" - self.assertEqual(os.path.exists(schema_path), True) - def test_nb2(self): """Test that a v2 notebook converted to v3 passes validation""" with self.fopen(u'test2.ipynb', u'r') as f: nb = read(f, u'json') - self.assertEqual(validate(nb), []) + validate(nb) self.assertEqual(isvalid(nb), True) def test_nb3(self): """Test that a v3 notebook passes validation""" with self.fopen(u'test3.ipynb', u'r') as f: nb = read(f, u'json') - self.assertEqual(validate(nb), []) + validate(nb) self.assertEqual(isvalid(nb), True) def test_invalid(self): @@ -52,22 +39,7 @@ class TestValidator(TestsBase): # - one cell has an invalid level with self.fopen(u'invalid.ipynb', u'r') as f: nb = read(f, u'json') - self.assertEqual(len(validate(nb)), 3) + with self.assertRaises(ValidationError): + validate(nb) self.assertEqual(isvalid(nb), False) - def test_resolve_ref(self): - """Test that references are correctly resolved""" - # make sure it resolves the ref correctly - json = {"abc": "def", "ghi": {"$ref": "/abc"}} - resolved = resolve_ref(json) - self.assertEqual(resolved, {"abc": "def", "ghi": "def"}) - - # make sure it throws an error if the ref is not by itself - json = {"abc": "def", "ghi": {"$ref": "/abc", "foo": "bar"}} - with self.assertRaises(SchemaError): - resolved = resolve_ref(json) - - # make sure it can handle json with no reference - json = {"abc": "def"} - resolved = resolve_ref(json) - self.assertEqual(resolved, json) diff --git a/IPython/nbformat/v3/nbbase.py b/IPython/nbformat/v3/nbbase.py index f27572d..42f9224 100644 --- a/IPython/nbformat/v3/nbbase.py +++ b/IPython/nbformat/v3/nbbase.py @@ -22,7 +22,7 @@ from IPython.utils.py3compat import cast_unicode, unicode_type # Change this when incrementing the nbformat version nbformat = 3 nbformat_minor = 0 -nbformat_schema = 'v3.withref.json' +nbformat_schema = 'nbformat.v3.schema.json' class NotebookNode(Struct): pass diff --git a/IPython/nbformat/v3/nbformat.v3.schema.json b/IPython/nbformat/v3/nbformat.v3.schema.json new file mode 100644 index 0000000..87e9615 --- /dev/null +++ b/IPython/nbformat/v3/nbformat.v3.schema.json @@ -0,0 +1,363 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "description": "IPython Notebook v3.0 JSON schema.", + "type": "object", + "additionalProperties": false, + "required": ["metadata", "nbformat_minor", "nbformat", "worksheets"], + "properties": { + "metadata": { + "description": "Notebook root-level metadata.", + "type": "object", + "additionalProperties": true, + "properties": { + "kernel_info": { + "description": "Kernel information.", + "type": "object", + "required": ["name", "language"], + "properties": { + "name": { + "description": "Name of the kernel specification.", + "type": "string" + }, + "language": { + "description": "The programming language which this kernel runs.", + "type": "string" + }, + "codemirror_mode": { + "description": "The codemirror mode to use for code in this language.", + "type": "string" + } + } + }, + "signature": { + "description": "Hash of the notebook.", + "type": "string" + } + } + }, + "nbformat_minor": { + "description": "Notebook format (minor number). Incremented for backward compatible changes to the notebook format.", + "type": "integer", + "minimum": 0 + }, + "nbformat": { + "description": "Notebook format (major number). Incremented between backwards incompatible changes to the notebook format.", + "type": "integer", + "minimum": 3, + "maximum": 3 + }, + "orig_nbformat": { + "description": "Original notebook format (major number) before converting the notebook between versions.", + "type": "integer", + "minimum": 1 + }, + "worksheets" : { + "description": "Array of worksheets", + "type": "array", + "items": {"$ref": "#/definitions/worksheet"} + } + }, + + "definitions": { + "worksheet": { + "additionalProperties": false, + "required" : ["cells"], + "properties":{ + "cells": { + "description": "Array of cells of the current notebook.", + "type": "array", + "items": { + "type": "object", + "oneOf": [ + {"$ref": "#/definitions/raw_cell"}, + {"$ref": "#/definitions/markdown_cell"}, + {"$ref": "#/definitions/heading_cell"}, + {"$ref": "#/definitions/code_cell"} + ] + } + }, + "metadata": { + "type": "object", + "description": "metadata of the current worksheet" + } + } + }, + "raw_cell": { + "description": "Notebook raw nbconvert cell.", + "type": "object", + "additionalProperties": false, + "required": ["cell_type", "source"], + "properties": { + "cell_type": { + "description": "String identifying the type of cell.", + "enum": ["raw"] + }, + "metadata": { + "description": "Cell-level metadata.", + "type": "object", + "additionalProperties": true, + "properties": { + "format": { + "description": "Raw cell metadata format for nbconvert.", + "type": "string" + }, + "name": {"$ref": "#/definitions/misc/metadata_name"}, + "tags": {"$ref": "#/definitions/misc/metadata_tags"} + } + }, + "source": {"$ref": "#/definitions/misc/source"} + } + }, + + "markdown_cell": { + "description": "Notebook markdown cell.", + "type": "object", + "additionalProperties": false, + "required": ["cell_type", "source"], + "properties": { + "cell_type": { + "description": "String identifying the type of cell.", + "enum": ["markdown"] + }, + "metadata": { + "description": "Cell-level metadata.", + "type": "object", + "properties": { + "name": {"$ref": "#/definitions/misc/metadata_name"}, + "tags": {"$ref": "#/definitions/misc/metadata_tags"} + }, + "additionalProperties": true + }, + "source": {"$ref": "#/definitions/misc/source"} + } + }, + + "heading_cell": { + "description": "Notebook heading cell.", + "type": "object", + "additionalProperties": false, + "required": ["cell_type", "source", "level"], + "properties": { + "cell_type": { + "description": "String identifying the type of cell.", + "enum": ["heading"] + }, + "metadata": { + "description": "Cell-level metadata.", + "type": "object", + "additionalProperties": true + }, + "source": {"$ref": "#/definitions/misc/source"}, + "level": { + "description": "Level of heading cells.", + "type": "integer", + "minimum": 1 + } + } + }, + + "code_cell": { + "description": "Notebook code cell.", + "type": "object", + "additionalProperties": false, + "required": ["cell_type", "input", "outputs", "collapsed", "language"], + "properties": { + "cell_type": { + "description": "String identifying the type of cell.", + "enum": ["code"] + }, + "language": { + "description": "The cell's language (always Python)", + "type": "string" + }, + "collapsed": { + "description": "Whether the cell is collapsed/expanded.", + "type": "boolean" + }, + "metadata": { + "description": "Cell-level metadata.", + "type": "object", + "additionalProperties": true + }, + "input": {"$ref": "#/definitions/misc/source"}, + "outputs": { + "description": "Execution, display, or stream outputs.", + "type": "array", + "items": {"$ref": "#/definitions/output"} + }, + "prompt_number": { + "description": "The code cell's prompt number. Will be null if the cell has not been run.", + "type": ["integer", "null"], + "minimum": 0 + } + } + }, + "output": { + "type": "object", + "oneOf": [ + {"$ref": "#/definitions/pyout"}, + {"$ref": "#/definitions/display_data"}, + {"$ref": "#/definitions/stream"}, + {"$ref": "#/definitions/pyerr"} + ] + }, + "pyout": { + "description": "Result of executing a code cell.", + "type": "object", + "additionalProperties": false, + "required": ["output_type", "prompt_number"], + "properties": { + "output_type": { + "description": "Type of cell output.", + "enum": ["pyout"] + }, + "prompt_number": { + "description": "A result's prompt number.", + "type": ["integer"], + "minimum": 0 + }, + "text": {"$ref": "#/definitions/misc/multiline_string"}, + "latex": {"$ref": "#/definitions/misc/multiline_string"}, + "png": {"$ref": "#/definitions/misc/multiline_string"}, + "jpeg": {"$ref": "#/definitions/misc/multiline_string"}, + "svg": {"$ref": "#/definitions/misc/multiline_string"}, + "html": {"$ref": "#/definitions/misc/multiline_string"}, + "javascript": {"$ref": "#/definitions/misc/multiline_string"}, + "json": {"$ref": "#/definitions/misc/multiline_string"}, + "pdf": {"$ref": "#/definitions/misc/multiline_string"}, + "metadata": {"$ref": "#/definitions/misc/output_metadata"} + }, + "patternProperties": { + "^[a-zA-Z0-9]+/[a-zA-Z0-9\\-\\+\\.]+$": { + "description": "mimetype output (e.g. text/plain), represented as either an array of strings or a string.", + "$ref": "#/definitions/misc/multiline_string" + } + } + }, + + "display_data": { + "description": "Data displayed as a result of code cell execution.", + "type": "object", + "additionalProperties": false, + "required": ["output_type"], + "properties": { + "output_type": { + "description": "Type of cell output.", + "enum": ["display_data"] + }, + "text": {"$ref": "#/definitions/misc/multiline_string"}, + "latex": {"$ref": "#/definitions/misc/multiline_string"}, + "png": {"$ref": "#/definitions/misc/multiline_string"}, + "jpeg": {"$ref": "#/definitions/misc/multiline_string"}, + "svg": {"$ref": "#/definitions/misc/multiline_string"}, + "html": {"$ref": "#/definitions/misc/multiline_string"}, + "javascript": {"$ref": "#/definitions/misc/multiline_string"}, + "json": {"$ref": "#/definitions/misc/multiline_string"}, + "pdf": {"$ref": "#/definitions/misc/multiline_string"}, + "metadata": {"$ref": "#/definitions/misc/output_metadata"} + }, + "patternProperties": { + "[a-zA-Z0-9]+/[a-zA-Z0-9\\-\\+\\.]+$": { + "description": "mimetype output (e.g. text/plain), represented as either an array of strings or a string.", + "$ref": "#/definitions/misc/multiline_string" + } + } + }, + + "stream": { + "description": "Stream output from a code cell.", + "type": "object", + "additionalProperties": false, + "required": ["output_type", "stream", "text"], + "properties": { + "output_type": { + "description": "Type of cell output.", + "enum": ["stream"] + }, + "stream": { + "description": "The stream type/destination.", + "type": "string" + }, + "text": { + "description": "The stream's text output, represented as an array of strings.", + "$ref": "#/definitions/misc/multiline_string" + } + } + }, + + "pyerr": { + "description": "Output of an error that occurred during code cell execution.", + "type": "object", + "additionalProperties": false, + "required": ["output_type", "ename", "evalue", "traceback"], + "properties": { + "output_type": { + "description": "Type of cell output.", + "enum": ["pyerr"] + }, + "metadata": {"$ref": "#/definitions/misc/output_metadata"}, + "ename": { + "description": "The name of the error.", + "type": "string" + }, + "evalue": { + "description": "The value, or message, of the error.", + "type": "string" + }, + "traceback": { + "description": "The error's traceback, represented as an array of strings.", + "type": "array", + "items": {"type": "string"} + } + } + }, + + "misc": { + "metadata_name": { + "description": "The cell's name. If present, must be a non-empty string.", + "type": "string", + "pattern": "^.+$" + }, + "metadata_tags": { + "description": "The cell's tags. Tags must be unique, and must not contain commas.", + "type": "array", + "uniqueItems": true, + "items": { + "type": "string", + "pattern": "^[^,]+$" + } + }, + "source": { + "description": "Contents of the cell, represented as an array of lines.", + "$ref": "#/definitions/misc/multiline_string" + }, + "prompt_number": { + "description": "The code cell's prompt number. Will be null if the cell has not been run.", + "type": ["integer", "null"], + "minimum": 0 + }, + "mimetype": { + "patternProperties": { + "^[a-zA-Z0-9\\-\\+]+/[a-zA-Z0-9\\-\\+]+": { + "description": "The cell's mimetype output (e.g. text/plain), represented as either an array of strings or a string.", + "$ref": "#/definitions/misc/multiline_string" + } + } + }, + "output_metadata": { + "description": "Cell output metadata.", + "type": "object", + "additionalProperties": true + }, + "multiline_string": { + "oneOf" : [ + {"type": "string"}, + { + "type": "array", + "items": {"type": "string"} + } + ] + } + } + } +} diff --git a/IPython/nbformat/v3/v3.withref.json b/IPython/nbformat/v3/v3.withref.json deleted file mode 100644 index 9a711a2..0000000 --- a/IPython/nbformat/v3/v3.withref.json +++ /dev/null @@ -1,178 +0,0 @@ -{ - "description": "custom json structure with references to generate notebook schema", - "notebook":{ - "type": "object", - "description": "notebook v3.0 root schema", - "$schema": "http://json-schema.org/draft-03/schema", - "id": "#notebook", - "required": true, - "additionalProperties": false, - "properties":{ - "metadata": { - "type": "object", - "id": "metadata", - "required": true, - "description": "the metadata atribute can contain any additionnal information", - "additionalProperties": true, - "properties":{ - "name": { - "id": "name", - "description": "the title of the notebook", - "type": "string", - "id": "name", - "required": true - } - } - }, - "nbformat_minor": { - "description": "Notebook format, minor number. Incremented for slight variation of notebook format.", - "type": "integer", - "minimum": 0, - "id": "nbformat_minor", - "required": true - }, - "nbformat": { - "description": "Notebook format, major number. Incremented between backward incompatible change is introduced.", - "type": "integer", - "minimum": 3, - "id": "nbformat", - "required": true - }, - "orig_nbformat": { - "description": "Original notebook format, major number.", - "type": "integer", - "minimum": 1, - "id": "orig_nbformat", - "required": false - }, - "worksheets": { - "description": "Array of worksheet, not used by the current implementation of ipython yet", - "type": "array", - "id": "worksheets", - "required": true, - "items": {"$ref": "/worksheet"} - } - } - }, - - "worksheet": { - "additionalProperties": false, - "properties":{ - "cells": { - "type": "array", - "$schema": "http://json-schema.org/draft-03/schema", - "description": "array of cells of the current worksheet", - "id": "#cells", - "required": true, - "items": {"$ref": "/any_cell"} - - }, - "metadata": { - "type": "object", - "description": "metadata of the current worksheet", - "id": "metadata", - "required": false - } - } - }, - - "text_cell": { - "type": "object", - "description": "scheme for text cel and childrenm (level only optionnal argument for HEader cell)", - "$schema": "http://json-schema.org/draft-03/schema", - "id": "#cell", - "required": true, - "additionalProperties": false, - "properties":{ - "cell_type": { - "type": "string", - "id": "cell_type", - "required": true - }, - "level": { - "type": "integer", - "minimum": 1, - "maximum": 6, - "id": "level", - "required": false - }, - "metadata": { - "type": "object", - "id": "metadata", - "required": false - }, - "source": { - "description": "for code cell, the source code", - "type": ["array", "string"], - "id": "source", - "required": true, - "items": - { - "type": "string", - "description": "each item represent one line of the source code written, terminated by \n", - "id": "0", - "required": true - } - } - } - - }, - - "any_cell": { - "description": "Meta cell type that match any cell type", - "type": [{"$ref": "/text_cell"},{"$ref":"/code_cell"}], - "$schema": "http://json-schema.org/draft-03/schema" - }, - - "code_cell":{ - "type": "object", - "$schema": "http://json-schema.org/draft-03/schema", - "description": "Cell used to execute code", - "id": "#cell", - "required": true, - "additionalProperties": false, - "properties":{ - "cell_type": { - "type": "string", - "id": "cell_type", - "required": true - }, - "metadata": { - "type": "object", - "id": "metadata", - "required": false - }, - "collapsed": { - "type": "boolean", - "required": true - }, - "input": { - "description": "user input for text cells", - "type": ["array", "string"], - "id": "input", - "required": true, - "items": - { - "type": "string", - "id": "input", - "required": true - } - }, - "outputs": { - "description": "output for code cell, to be definied", - "required": true, - "type": "array" - }, - "prompt_number": { - "type": ["integer","null"], - "required": false, - "minimum": 0 - }, - "language": { - "type": "string", - "required": true - } - } - - } -} diff --git a/IPython/nbformat/validator.py b/IPython/nbformat/validator.py index e0d486a..7d5d801 100644 --- a/IPython/nbformat/validator.py +++ b/IPython/nbformat/validator.py @@ -1,112 +1,72 @@ +# Copyright (c) IPython Development Team. +# Distributed under the terms of the Modified BSD License. + from __future__ import print_function import json import os try: - from jsonschema import SchemaError - from jsonschema import Draft3Validator as Validator + from jsonschema import ValidationError + from jsonschema import Draft4Validator as Validator except ImportError as e: verbose_msg = """ - IPython depends on the jsonschema package: https://pypi.python.org/pypi/jsonschema + IPython notebook format depends on the jsonschema package: - Please install it first. - """ - raise ImportError(str(e) + verbose_msg) - -try: - import jsonpointer as jsonpointer -except ImportError as e: - verbose_msg = """ - - IPython depends on the jsonpointer package: https://pypi.python.org/pypi/jsonpointer + https://pypi.python.org/pypi/jsonschema Please install it first. """ raise ImportError(str(e) + verbose_msg) -from IPython.utils.py3compat import iteritems +from IPython.utils.importstring import import_item -from .current import nbformat, nbformat_schema -schema_path = os.path.join( - os.path.dirname(__file__), "v%d" % nbformat, nbformat_schema) +validators = {} +def get_validator(version=None): + """Load the JSON schema into a Validator""" + if version is None: + from .current import nbformat as version -def isvalid(nbjson): + if version not in validators: + v = import_item("IPython.nbformat.v%s" % version) + schema_path = os.path.join(os.path.dirname(v.__file__), v.nbformat_schema) + with open(schema_path) as f: + schema_json = json.load(f) + validators[version] = Validator(schema_json) + return validators[version] + +def isvalid(nbjson, ref=None, version=None): """Checks whether the given notebook JSON conforms to the current notebook format schema. Returns True if the JSON is valid, and False otherwise. To see the individual errors that were encountered, please use the `validate` function instead. - """ - - errors = validate(nbjson) - return errors == [] + try: + validate(nbjson, ref, version) + except ValidationError: + return False + else: + return True -def validate(nbjson): +def validate(nbjson, ref=None, version=None): """Checks whether the given notebook JSON conforms to the current - notebook format schema, and returns the list of errors. + notebook format schema. + Raises ValidationError if not valid. """ + if version is None: + from .current import nbformat + version = nbjson.get('nbformat', nbformat) - # load the schema file - with open(schema_path, 'r') as fh: - schema_json = json.load(fh) - - # resolve internal references - schema = resolve_ref(schema_json) - schema = jsonpointer.resolve_pointer(schema, '/notebook') - - # count how many errors there are - v = Validator(schema) - errors = list(v.iter_errors(nbjson)) - return errors - - -def resolve_ref(json, schema=None): - """Resolve internal references within the given JSON. This essentially - means that dictionaries of this form: - - {"$ref": "/somepointer"} - - will be replaced with the resolved reference to `/somepointer`. - This only supports local reference to the same JSON file. - - """ + validator = get_validator(version) - if not schema: - schema = json - - # if it's a list, resolve references for each item in the list - if type(json) is list: - resolved = [] - for item in json: - resolved.append(resolve_ref(item, schema=schema)) - - # if it's a dictionary, resolve references for each item in the - # dictionary - elif type(json) is dict: - resolved = {} - for key, ref in iteritems(json): - - # if the key is equal to $ref, then replace the entire - # dictionary with the resolved value - if key == '$ref': - if len(json) != 1: - raise SchemaError( - "objects containing a $ref should only have one item") - pointer = jsonpointer.resolve_pointer(schema, ref) - resolved = resolve_ref(pointer, schema=schema) - - else: - resolved[key] = resolve_ref(ref, schema=schema) - - # otherwise it's a normal object, so just return it + if ref: + return validator.validate(nbjson, {'$ref' : '#/definitions/%s' % ref}) else: - resolved = json + return validator.validate(nbjson) - return resolved diff --git a/IPython/testing/iptestcontroller.py b/IPython/testing/iptestcontroller.py index 12ee610..3a04713 100644 --- a/IPython/testing/iptestcontroller.py +++ b/IPython/testing/iptestcontroller.py @@ -218,7 +218,7 @@ def all_js_groups(): class JSController(TestController): """Run CasperJS tests """ requirements = ['zmq', 'tornado', 'jinja2', 'casperjs', 'sqlite3', - 'jsonschema', 'jsonpointer'] + 'jsonschema'] display_slimer_output = False def __init__(self, section, xunit=True, engine='phantomjs'): diff --git a/setup.py b/setup.py index 6733400..7d506da 100755 --- a/setup.py +++ b/setup.py @@ -275,7 +275,7 @@ extras_require = dict( doc = ['Sphinx>=1.1', 'numpydoc'], test = ['nose>=0.10.1'], terminal = [], - nbformat = ['jsonschema>=2.0', 'jsonpointer>=1.3'], + nbformat = ['jsonschema>=2.0'], notebook = ['tornado>=3.1', 'pyzmq>=2.1.11', 'jinja2', 'pygments', 'mistune>=0.3.1'], nbconvert = ['pygments', 'jinja2', 'mistune>=0.3.1'] ) diff --git a/setupbase.py b/setupbase.py index d0495be..2516e4f 100644 --- a/setupbase.py +++ b/setupbase.py @@ -194,7 +194,10 @@ def find_package_data(): 'preprocessors/tests/files/*.*', ], 'IPython.nbconvert.filters' : ['marked.js'], - 'IPython.nbformat' : ['tests/*.ipynb','v3/v3.withref.json'] + 'IPython.nbformat' : [ + 'tests/*.ipynb', + 'v3/nbformat.v3.schema.json', + ] } return package_data