#!/usr/bin/env python """Convert IPython notebooks to other formats, such as ReST, and HTML. Example: ./nbconvert.py --format rst file.ipynb Produces 'file.rst', along with auto-generated figure files called nb_figure_NN.png. """ #----------------------------------------------------------------------------- # Imports #----------------------------------------------------------------------------- from __future__ import print_function # Stdlib import codecs import io import logging import os import pprint import re import subprocess import sys import json import copy from shutil import rmtree from markdown import markdown inkscape = 'inkscape' if sys.platform == 'darwin': inkscape = '/Applications/Inkscape.app/Contents/Resources/bin/inkscape' if not os.path.exists(inkscape): inkscape = None # From IPython from IPython.external import argparse from IPython.nbformat import current as nbformat from IPython.utils.text import indent from IPython.nbformat.v3.nbjson import BytesEncoder from IPython.utils import path, py3compat # local from decorators import DocInherit from lexers import IPythonLexer #----------------------------------------------------------------------------- # Utility functions #----------------------------------------------------------------------------- def DocInherit(f): return f def remove_fake_files_url(cell): """Remove from the cell source the /files/ pseudo-path we use. """ src = cell.source cell.source = src.replace('/files/', '') # ANSI color functions: def remove_ansi(src): """Strip all ANSI color escape sequences from input string. Parameters ---------- src : string Returns ------- string """ return re.sub(r'\033\[(0|\d;\d\d)m', '', src) def ansi2html(txt): """Render ANSI colors as HTML colors This is equivalent to util.fixConsole in utils.js Parameters ---------- txt : string Returns ------- string """ ansi_colormap = { '30': 'ansiblack', '31': 'ansired', '32': 'ansigreen', '33': 'ansiyellow', '34': 'ansiblue', '35': 'ansipurple', '36': 'ansicyan', '37': 'ansigrey', '01': 'ansibold', } # do ampersand first txt = txt.replace('&', '&') html_escapes = { '<': '<', '>': '>', "'": ''', '"': '"', '`': '`', } for c, escape in html_escapes.iteritems(): txt = txt.replace(c, escape) ansi_re = re.compile('\x1b' + r'\[([\dA-Fa-f;]*?)m') m = ansi_re.search(txt) opened = False cmds = [] opener = '' closer = '' while m: cmds = m.groups()[0].split(';') closer = '' if opened else '' opened = len(cmds) > 1 or cmds[0] != '0'*len(cmds[0]); classes = [] for cmd in cmds: if cmd in ansi_colormap: classes.append(ansi_colormap.get(cmd)) if classes: opener = '' % (' '.join(classes)) else: opener = '' txt = re.sub(ansi_re, closer + opener, txt, 1) m = ansi_re.search(txt) if opened: txt += '' return txt # Pandoc-dependent code def markdown2latex(src): """Convert a markdown string to LaTeX via pandoc. This function will raise an error if pandoc is not installed. Any error messages generated by pandoc are printed to stderr. Parameters ---------- src : string Input string, assumed to be valid markdown. Returns ------- out : string Output as returned by pandoc. """ p = subprocess.Popen('pandoc -f markdown -t latex'.split(), stdin=subprocess.PIPE, stdout=subprocess.PIPE) out, err = p.communicate(src.encode('utf-8')) if err: print(err, file=sys.stderr) #print('*'*20+'\n', out, '\n'+'*'*20) # dbg return unicode(out,'utf-8') def markdown2rst(src): """Convert a markdown string to LaTeX via pandoc. This function will raise an error if pandoc is not installed. Any error messages generated by pandoc are printed to stderr. Parameters ---------- src : string Input string, assumed to be valid markdown. Returns ------- out : string Output as returned by pandoc. """ p = subprocess.Popen('pandoc -f markdown -t rst'.split(), stdin=subprocess.PIPE, stdout=subprocess.PIPE) out, err = p.communicate(src.encode('utf-8')) if err: print(err, file=sys.stderr) #print('*'*20+'\n', out, '\n'+'*'*20) # dbg return unicode(out,'utf-8') def rst_directive(directive, text=''): out = [directive, ''] if text: out.extend([indent(text), '']) return out def coalesce_streams(outputs): """merge consecutive sequences of stream output into single stream to prevent extra newlines inserted at flush calls TODO: handle \r deletion """ new_outputs = [] last = outputs[0] new_outputs = [last] for output in outputs[1:]: if (output.output_type == 'stream' and last.output_type == 'stream' and last.stream == output.stream ): last.text += output.text else: new_outputs.append(output) return new_outputs #----------------------------------------------------------------------------- # Class declarations #----------------------------------------------------------------------------- class ConversionException(Exception): pass class Converter(object): default_encoding = 'utf-8' extension = str() figures_counter = 0 infile = str() infile_dir = str() infile_root = str() files_dir = str() with_preamble = True user_preamble = None output = str() raw_as_verbatim = False def __init__(self, infile): self.infile = infile self.infile_dir, infile_root = os.path.split(infile) infile_root = os.path.splitext(infile_root)[0] files_dir = os.path.join(self.infile_dir, infile_root + '_files') if not os.path.isdir(files_dir): os.mkdir(files_dir) self.infile_root = infile_root self.files_dir = files_dir self.outbase = os.path.join(self.infile_dir, infile_root) def __del__(self): if not os.listdir(self.files_dir): os.rmdir(self.files_dir) def dispatch(self, cell_type): """return cell_type dependent render method, for example render_code """ return getattr(self, 'render_' + cell_type, self.render_unknown) def dispatch_display_format(self, format): """return output_type dependent render method, for example render_output_text """ return getattr(self, 'render_display_format_' + format, self.render_unknown_display) def convert(self, cell_separator='\n'): lines = [] lines.extend(self.optional_header()) converted_cells = [] for worksheet in self.nb.worksheets: for cell in worksheet.cells: #print(cell.cell_type) # dbg conv_fn = self.dispatch(cell.cell_type) if cell.cell_type in ('markdown', 'raw'): remove_fake_files_url(cell) converted_cells.append('\n'.join(conv_fn(cell))) cell_lines = cell_separator.join(converted_cells).split('\n') lines.extend(cell_lines) lines.extend(self.optional_footer()) return u'\n'.join(lines) def render(self): "read, convert, and save self.infile" if not hasattr(self, 'nb'): self.read() self.output = self.convert() return self.save() def read(self): "read and parse notebook into NotebookNode called self.nb" with open(self.infile) as f: self.nb = nbformat.read(f, 'json') def save(self, outfile=None, encoding=None): "read and parse notebook into self.nb" if outfile is None: outfile = self.outbase + '.' + self.extension if encoding is None: encoding = self.default_encoding with io.open(outfile, 'w', encoding=encoding) as f: f.write(self.output) return os.path.abspath(outfile) def optional_header(self): return [] def optional_footer(self): return [] def _new_figure(self, data, fmt): """Create a new figure file in the given format. Returns a path relative to the input file. """ figname = '%s_fig_%02i.%s' % (self.infile_root, self.figures_counter, fmt) self.figures_counter += 1 fullname = os.path.join(self.files_dir, figname) # Binary files are base64-encoded, SVG is already XML if fmt in ('png', 'jpg', 'pdf'): data = data.decode('base64') fopen = lambda fname: open(fname, 'wb') else: fopen = lambda fname: codecs.open(fname, 'wb', self.default_encoding) with fopen(fullname) as f: f.write(data) return fullname def render_heading(self, cell): """convert a heading cell Returns list.""" raise NotImplementedError def render_code(self, cell): """Convert a code cell Returns list.""" raise NotImplementedError def render_markdown(self, cell): """convert a markdown cell Returns list.""" raise NotImplementedError def _img_lines(self, img_file): """Return list of lines to include an image file.""" # Note: subclasses may choose to implement format-specific _FMT_lines # methods if they so choose (FMT in {png, svg, jpg, pdf}). raise NotImplementedError def render_display_data(self, output): """convert display data from the output of a code cell Returns list. """ lines = [] for fmt in output.keys(): if fmt in ['png', 'svg', 'jpg', 'pdf']: img_file = self._new_figure(output[fmt], fmt) # Subclasses can have format-specific render functions (e.g., # latex has to auto-convert all SVG to PDF first). lines_fun = getattr(self, '_%s_lines' % fmt, None) if not lines_fun: lines_fun = self._img_lines lines.extend(lines_fun(img_file)) elif fmt != 'output_type': conv_fn = self.dispatch_display_format(fmt) lines.extend(conv_fn(output)) return lines def render_raw(self, cell): """convert a cell with raw text Returns list.""" raise NotImplementedError def render_unknown(self, cell): """Render cells of unkown type Returns list.""" data = pprint.pformat(cell) logging.warning('Unknown cell: %s' % cell.cell_type) return self._unknown_lines(data) def render_unknown_display(self, output, type): """Render cells of unkown type Returns list.""" data = pprint.pformat(output) logging.warning('Unknown output: %s' % output.output_type) return self._unknown_lines(data) def render_stream(self, output): """render the stream part of an output Returns list. Identical to render_display_format_text """ return self.render_display_format_text(output) def render_pyout(self, output): """convert pyout part of a code cell Returns list.""" raise NotImplementedError def render_pyerr(self, output): """convert pyerr part of a code cell Returns list.""" raise NotImplementedError def _unknown_lines(self, data): """Return list of lines for an unknown cell. Parameters ---------- data : str The content of the unknown data as a single string. """ raise NotImplementedError # These are the possible format types in an output node def render_display_format_text(self, output): """render the text part of an output Returns list. """ raise NotImplementedError def render_display_format_html(self, output): """render the html part of an output Returns list. """ raise NotImplementedError def render_display_format_latex(self, output): """render the latex part of an output Returns list. """ raise NotImplementedError def render_display_format_json(self, output): """render the json part of an output Returns list. """ raise NotImplementedError def render_display_format_javascript(self, output): """render the javascript part of an output Returns list. """ raise NotImplementedError class ConverterRST(Converter): extension = 'rst' heading_level = {1: '=', 2: '-', 3: '`', 4: '\'', 5: '.', 6: '~'} @DocInherit def render_heading(self, cell): marker = self.heading_level[cell.level] return ['{0}\n{1}\n'.format(cell.source, marker * len(cell.source))] @DocInherit def render_code(self, cell): if not cell.input: return [] lines = ['In[%s]:' % cell.prompt_number, ''] lines.extend(rst_directive('.. code:: python', cell.input)) for output in cell.outputs: conv_fn = self.dispatch(output.output_type) lines.extend(conv_fn(output)) return lines @DocInherit def render_markdown(self, cell): #return [cell.source] return [markdown2rst(cell.source)] @DocInherit def render_raw(self, cell): if self.raw_as_verbatim: return ['::', '', indent(cell.source), ''] else: return [cell.source] @DocInherit def render_pyout(self, output): lines = ['Out[%s]:' % output.prompt_number, ''] # output is a dictionary like object with type as a key if 'latex' in output: lines.extend(rst_directive('.. math::', output.latex)) if 'text' in output: lines.extend(rst_directive('.. parsed-literal::', output.text)) return lines @DocInherit def render_pyerr(self, output): # Note: a traceback is a *list* of frames. return ['::', '', indent(remove_ansi('\n'.join(output.traceback))), ''] @DocInherit def _img_lines(self, img_file): return ['.. image:: %s' % img_file, ''] @DocInherit def render_display_format_text(self, output): return rst_directive('.. parsed-literal::', output.text) @DocInherit def _unknown_lines(self, data): return rst_directive('.. warning:: Unknown cell') + [data] @DocInherit def render_display_format_html(self, output): return rst_directive('.. raw:: html', output.html) @DocInherit def render_display_format_latex(self, output): return rst_directive('.. math::', output.latex) @DocInherit def render_display_format_json(self, output): return rst_directive('.. raw:: json', output.json) @DocInherit def render_display_format_javascript(self, output): return rst_directive('.. raw:: javascript', output.javascript) def highlight(src, lang='ipython'): """Return a syntax-highlighted version of the input source. """ from pygments import highlight from pygments.lexers import get_lexer_by_name from pygments.formatters import HtmlFormatter if lang == 'ipython': lexer = IPythonLexer() else: lexer = get_lexer_by_name(lang, stripall=True) return highlight(src, lexer, HtmlFormatter()) class ConverterMarkdown(Converter): extension = 'md' def __init__(self, infile, highlight_source=True, show_prompts=False, inline_prompt=False): super(ConverterMarkdown, self).__init__(infile) self.highlight_source = highlight_source self.show_prompts = show_prompts self.inline_prompt = inline_prompt @DocInherit def render_heading(self, cell): return ['{0} {1}'.format('#'*cell.level, cell.source), ''] @DocInherit def render_code(self, cell): if not cell.input: return [] lines = [] if self.show_prompts and not self.inline_prompt: lines.extend(['*In[%s]:*' % cell.prompt_number, '']) if self.show_prompts and self.inline_prompt: prompt = 'In[%s]: ' % cell.prompt_number input_lines = cell.input.split('\n') src = prompt + input_lines[0] + '\n' + indent('\n'.join(input_lines[1:]), nspaces=len(prompt)) else: src = cell.input src = highlight(src) if self.highlight_source else indent(src) lines.extend([src, '']) if cell.outputs and self.show_prompts and not self.inline_prompt: lines.extend(['*Out[%s]:*' % cell.prompt_number, '']) for output in cell.outputs: conv_fn = self.dispatch(output.output_type) lines.extend(conv_fn(output)) #lines.append('----') lines.append('') return lines @DocInherit def render_markdown(self, cell): return [cell.source, ''] @DocInherit def render_raw(self, cell): if self.raw_as_verbatim: return [indent(cell.source), ''] else: return [cell.source, ''] @DocInherit def render_pyout(self, output): lines = [] ## if 'text' in output: ## lines.extend(['*Out[%s]:*' % output.prompt_number, '']) # output is a dictionary like object with type as a key if 'latex' in output: pass if 'text' in output: lines.extend(['
', indent(output.text), '
']) lines.append('') return lines @DocInherit def render_pyerr(self, output): # Note: a traceback is a *list* of frames. return [indent(remove_ansi('\n'.join(output.traceback))), ''] @DocInherit def _img_lines(self, img_file): return ['', '![](%s)' % img_file, ''] @DocInherit def render_display_format_text(self, output): return [indent(output.text)] @DocInherit def _unknown_lines(self, data): return ['Warning: Unknown cell', data] @DocInherit def render_display_format_html(self, output): return [output.html] @DocInherit def render_display_format_latex(self, output): return ['LaTeX::', indent(output.latex)] @DocInherit def render_display_format_json(self, output): return ['JSON:', indent(output.json)] @DocInherit def render_display_format_javascript(self, output): return ['JavaScript:', indent(output.javascript)] def return_list(x): """Ensure that x is returned as a list or inside one""" return x if isinstance(x, list) else [x] # decorators for HTML output def output_container(f): """add a prompt-area next to an output""" def wrapped(self, output): rendered = f(self, output) if not rendered: # empty output return [] lines = [] lines.append('
') lines.extend(self._out_prompt(output)) classes = "output_subarea output_%s" % output.output_type if output.output_type == 'stream': classes += " output_%s" % output.stream lines.append('
' % classes) lines.extend(rendered) lines.append('
') # subarea lines.append('
') # output_area return lines return wrapped def text_cell(f): """wrap text cells in appropriate divs""" def wrapped(self, cell): rendered = f(self, cell) classes = "text_cell_render border-box-sizing rendered_html" lines = ['
' % classes] + rendered + ['
'] return lines return wrapped class ConverterHTML(Converter): extension = 'html' def in_tag(self, tag, src, attrs=None): """Return a list of elements bracketed by the given tag""" attr_s = '' if attrs is None else \ ' '.join( "%s=%s" % (attr, value) for attr, value in attrs.iteritems() ) return ['<%s %s>' % (tag, attr_s), src, '' % tag] def _ansi_colored(self, text): return ['
%s
' % ansi2html(text)] def _stylesheet(self, fname): with io.open(fname, encoding='utf-8') as f: s = f.read() return self.in_tag('style', s, dict(type='"text/css"')) def _out_prompt(self, output): if output.output_type == 'pyout': n = output.prompt_number if output.prompt_number is not None else ' ' content = 'Out [%s]:' % n else: content = '' return ['
%s
' % content] def header_body(self): """Return the body of the header as a list of strings.""" from pygments.formatters import HtmlFormatter header = [] static = os.path.join(path.get_ipython_package_dir(), 'frontend', 'html', 'notebook', 'static', ) here = os.path.split(os.path.abspath(__file__))[0] css = os.path.join(static, 'css') for sheet in [ # do we need jquery and prettify? # os.path.join(static, 'jquery', 'css', 'themes', 'base', 'jquery-ui.min.css'), # os.path.join(static, 'prettify', 'prettify.css'), os.path.join(css, 'boilerplate.css'), os.path.join(css, 'fbm.css'), os.path.join(css, 'notebook.css'), os.path.join(css, 'renderedhtml.css'), # our overrides: os.path.join(here, 'css', 'static_html.css'), ]: header.extend(self._stylesheet(sheet)) # pygments css pygments_css = HtmlFormatter().get_style_defs('.highlight') header.extend(['']) header.extend(self.in_tag('style', pygments_css, dict(type='"text/css"'))) # TODO: this should be allowed to use local mathjax: header.extend(self.in_tag('script', '', {'type':'"text/javascript"', 'src': '"https://c328740.ssl.cf1.rackcdn.com/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"', })) with io.open(os.path.join(here, 'js', 'initmathjax.js'), encoding='utf-8') as f: header.extend(self.in_tag('script', f.read(), {'type': '"text/javascript"'})) return header def optional_header(self): return ['', ''] + self.header_body() + \ ['', ''] def optional_footer(self): return ['', ''] @DocInherit @text_cell def render_heading(self, cell): marker = cell.level return [u'\n {0}\n'.format(cell.source, marker)] @DocInherit def render_code(self, cell): if not cell.input: return [] lines = ['
'] lines.append('
') n = cell.prompt_number if getattr(cell, 'prompt_number', None) is not None else ' ' lines.append('
In [%s]:
' % n) lines.append('
') lines.append(highlight(cell.input)) lines.append('
') # input_area lines.append('
') # input if cell.outputs: lines.append('
') lines.append('
') for output in coalesce_streams(cell.outputs): conv_fn = self.dispatch(output.output_type) lines.extend(conv_fn(output)) lines.append('
') # output lines.append('
') # output_wrapper lines.append('
') # cell return lines @DocInherit @text_cell def render_markdown(self, cell): return [markdown(cell.source)] @DocInherit def render_raw(self, cell): if self.raw_as_verbatim: return self.in_tag('pre', cell.source) else: return [cell.source] @DocInherit @output_container def render_pyout(self, output): for fmt in ['html', 'latex', 'png', 'jpeg', 'svg', 'text']: if fmt in output: conv_fn = self.dispatch_display_format(fmt) return conv_fn(output) return [] render_display_data = render_pyout @DocInherit @output_container def render_stream(self, output): return self._ansi_colored(output.text) @DocInherit @output_container def render_pyerr(self, output): # Note: a traceback is a *list* of frames. # lines = [] # stb = return self._ansi_colored('\n'.join(output.traceback)) @DocInherit def _img_lines(self, img_file): return ['' % img_file, ''] @DocInherit def _unknown_lines(self, data): return ['

Warning:: Unknown cell

'] + self.in_tag('pre', data) @DocInherit def render_display_format_png(self, output): return ['' % output.png] @DocInherit def render_display_format_svg(self, output): return [output.svg] @DocInherit def render_display_format_jpeg(self, output): return ['' % output.jpeg] @DocInherit def render_display_format_text(self, output): return self._ansi_colored(output.text) @DocInherit def render_display_format_html(self, output): return [output.html] @DocInherit def render_display_format_latex(self, output): return [output.latex] @DocInherit def render_display_format_json(self, output): # html ignores json return [] @DocInherit def render_display_format_javascript(self, output): return [output.javascript] class ConverterBloggerHTML(ConverterHTML): """Convert a notebook to html suitable for easy pasting into Blogger. It generates an html file that has *only* the pure HTML contents, and a separate file with `_header` appended to the name with all header contents. Typically, the header file only needs to be used once when setting up a blog, as the CSS for all posts is stored in a single location in Blogger. """ def optional_header(self): with io.open(self.outbase + '_header.html', 'w', encoding=self.default_encoding) as f: f.write('\n'.join(self.header_body())) return [] def optional_footer(self): return [] class ConverterLaTeX(Converter): """Converts a notebook to a .tex file suitable for pdflatex. Note: this converter *needs*: - `pandoc`: for all conversion of markdown cells. If your notebook only has Raw cells, pandoc will not be needed. - `inkscape`: if your notebook has SVG figures. These need to be converted to PDF before inclusion in the TeX file, as LaTeX doesn't understand SVG natively. You will in general obtain much better final PDF results if you configure the matplotlib backend to create SVG output with %config InlineBackend.figure_format = 'svg' (or set the equivalent flag at startup or in your configuration profile). """ extension = 'tex' documentclass = 'article' documentclass_options = '11pt,english' heading_map = {1: r'\section', 2: r'\subsection', 3: r'\subsubsection', 4: r'\paragraph', 5: r'\subparagraph', 6: r'\subparagraph'} def in_env(self, environment, lines): """Return list of environment lines for input lines Parameters ---------- env : string Name of the environment to bracket with begin/end. lines: """ out = [ur'\begin{%s}' % environment] if isinstance(lines, basestring): out.append(lines) else: # list out.extend(lines) out.append(ur'\end{%s}' % environment) return out def convert(self): # The main body is done by the logic in the parent class, and that's # all we need if preamble support has been turned off. body = super(ConverterLaTeX, self).convert() if not self.with_preamble: return body # But if preamble is on, then we need to construct a proper, standalone # tex file. # Tag the document at the top and set latex class final = [ r'%% This file was auto-generated by IPython, do NOT edit', r'%% Conversion from the original notebook file:', r'%% {0}'.format(self.infile), r'%%', r'\documentclass[%s]{%s}' % (self.documentclass_options, self.documentclass), '', ] # Load our own preamble, which is stored next to the main file. We # need to be careful in case the script entry point is a symlink myfile = __file__ if not os.path.islink(__file__) else \ os.readlink(__file__) with open(os.path.join(os.path.dirname(myfile), 'preamble.tex')) as f: final.append(f.read()) # Load any additional user-supplied preamble if self.user_preamble: final.extend(['', '%% Adding user preamble from file:', '%% {0}'.format(self.user_preamble), '']) with open(self.user_preamble) as f: final.append(f.read()) # Include document body final.extend([ r'\begin{document}', '', body, r'\end{document}', '']) # Retun value must be a string return '\n'.join(final) @DocInherit def render_heading(self, cell): marker = self.heading_map[cell.level] return ['%s{%s}' % (marker, cell.source) ] @DocInherit def render_code(self, cell): if not cell.input: return [] # Cell codes first carry input code, we use lstlisting for that lines = [ur'\begin{codecell}'] lines.extend(self.in_env('codeinput', self.in_env('lstlisting', cell.input))) outlines = [] for output in cell.outputs: conv_fn = self.dispatch(output.output_type) outlines.extend(conv_fn(output)) # And then output of many possible types; use a frame for all of it. if outlines: lines.extend(self.in_env('codeoutput', outlines)) lines.append(ur'\end{codecell}') return lines @DocInherit def _img_lines(self, img_file): return self.in_env('center', [r'\includegraphics[width=6in]{%s}' % img_file, r'\par']) def _svg_lines(self, img_file): base_file = os.path.splitext(img_file)[0] pdf_file = base_file + '.pdf' subprocess.check_call([ inkscape, '--export-pdf=%s' % pdf_file, img_file]) return self._img_lines(pdf_file) @DocInherit def render_markdown(self, cell): return [markdown2latex(cell.source)] @DocInherit def render_pyout(self, output): lines = [] # output is a dictionary like object with type as a key if 'latex' in output: lines.extend(output.latex) if 'text' in output: lines.extend(self.in_env('verbatim', output.text)) return lines @DocInherit def render_pyerr(self, output): # Note: a traceback is a *list* of frames. return self.in_env('traceback', self.in_env('verbatim', remove_ansi('\n'.join(output.traceback)))) @DocInherit def render_raw(self, cell): if self.raw_as_verbatim: return self.in_env('verbatim', cell.source) else: return [cell.source] @DocInherit def _unknown_lines(self, data): return [r'{\vspace{5mm}\bf WARNING:: unknown cell:}'] + \ self.in_env('verbatim', data) @DocInherit def render_display_format_text(self, output): lines = [] if 'text' in output: lines.extend(self.in_env('verbatim', output.text.strip())) return lines @DocInherit def render_display_format_html(self, output): return [] @DocInherit def render_display_format_latex(self, output): if type(output.latex) == type([]): return output.latex return [output.latex] @DocInherit def render_display_format_json(self, output): # latex ignores json return [] @DocInherit def render_display_format_javascript(self, output): # latex ignores javascript return [] class ConverterNotebook(Converter): """ A converter that is essentially a null-op. This exists so it can be subclassed for custom handlers of .ipynb files that create new .ipynb files. What distinguishes this from JSONWriter is that subclasses can specify what to do with each type of cell. Writes out a notebook file. """ extension = 'ipynb' def __init__(self, infile, outbase): Converter.__init__(self, infile) self.outbase = outbase rmtree(self.files_dir) def convert(self): return json.dumps(json.loads(Converter.convert(self, ',')), indent=1, sort_keys=True) def optional_header(self): s = \ """{ "metadata": { "name": "%(name)s" }, "nbformat": 3, "worksheets": [ { "cells": [""" % {'name':self.outbase} return s.split('\n') def optional_footer(self): s = \ """] } ] }""" return s.split('\n') @DocInherit def render_heading(self, cell): return cell_to_lines(cell) @DocInherit def render_code(self, cell): return cell_to_lines(cell) @DocInherit def render_markdown(self, cell): return cell_to_lines(cell) @DocInherit def render_raw(self, cell): return cell_to_lines(cell) @DocInherit def render_pyout(self, output): return cell_to_lines(output) @DocInherit def render_pyerr(self, output): return cell_to_lines(output) @DocInherit def render_display_format_text(self, output): return [output.text] @DocInherit def render_display_format_html(self, output): return [output.html] @DocInherit def render_display_format_latex(self, output): return [output.latex] @DocInherit def render_display_format_json(self, output): return [output.json] @DocInherit def render_display_format_javascript(self, output): return [output.javascript] class ConverterPy(Converter): """ A converter that takes a notebook and converts it to a .py file. What distinguishes this from PyWriter and PyReader in IPython.nbformat is that subclasses can specify what to do with each type of cell. Additionally, unlike PyWriter, this does not preserve the '# ' opening and closing comments style comments in favor of a cleaner looking python program. Note: Even though this produces a .py file, it is not guaranteed to be valid python file, since the notebook may be using magics and even cell magics. """ extension = 'py' def __init__(self, infile, show_prompts=True, show_output=True): super(ConverterPy, self).__init__(infile) self.show_prompts = show_prompts self.show_output = show_output @staticmethod def comment(input): "returns every line in input as commented out" return "# "+input.replace("\n", "\n# ") @DocInherit def render_heading(self, cell): return ['#{0} {1}'.format('#'*cell.level, cell.source), ''] @DocInherit def render_code(self, cell): if not cell.input: return [] lines = [] if self.show_prompts: lines.extend(['# In[%s]:' % cell.prompt_number]) src = cell.input lines.extend([src, '']) if self.show_output: if cell.outputs : lines.extend(['# Out[%s]:' % cell.prompt_number]) for output in cell.outputs: conv_fn = self.dispatch(output.output_type) lines.extend(conv_fn(output)) return lines @DocInherit def render_markdown(self, cell): return [self.comment(cell.source), ''] @DocInherit def render_raw(self, cell): if self.raw_as_verbatim: return [self.comment(indent(cell.source)), ''] else: return [self.comment(cell.source), ''] @DocInherit def render_pyout(self, output): lines = [] ## if 'text' in output: ## lines.extend(['*Out[%s]:*' % output.prompt_number, '']) # output is a dictionary like object with type as a key if 'latex' in output: pass if 'text' in output: lines.extend([self.comment(indent(output.text)), '']) lines.append('') return lines @DocInherit def render_pyerr(self, output): # Note: a traceback is a *list* of frames. return [indent(remove_ansi('\n'.join(output.traceback))), ''] @DocInherit def _img_lines(self, img_file): return [ self.comment('image file: %s' % img_file), ''] @DocInherit def render_display_format_text(self, output): return [self.comment(indent(output.text))] @DocInherit def _unknown_lines(self, data): return [self.comment('Warning: Unknown cell'+ str(data))] @DocInherit def render_display_format_html(self, output): return [self.comment(output.html)] @DocInherit def render_display_format_latex(self, output): return [] @DocInherit def render_display_format_json(self, output): return [] @DocInherit def render_display_format_javascript(self, output): return [] #----------------------------------------------------------------------------- # Standalone conversion functions #----------------------------------------------------------------------------- def rst2simplehtml(infile): """Convert a rst file to simplified html suitable for blogger. This just runs rst2html with certain parameters to produce really simple html and strips the document header, so the resulting file can be easily pasted into a blogger edit window. """ # This is the template for the rst2html call that produces the cleanest, # simplest html I could find. This should help in making it easier to # paste into the blogspot html window, though I'm still having problems # with linebreaks there... cmd_template = ("rst2html --link-stylesheet --no-xml-declaration " "--no-generator --no-datestamp --no-source-link " "--no-toc-backlinks --no-section-numbering " "--strip-comments ") cmd = "%s %s" % (cmd_template, infile) proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) html, stderr = proc.communicate() if stderr: raise IOError(stderr) # Make an iterator so breaking out holds state. Our implementation of # searching for the html body below is basically a trivial little state # machine, so we need this. walker = iter(html.splitlines()) # Find start of main text, break out to then print until we find end /div. # This may only work if there's a real title defined so we get a 'div class' # tag, I haven't really tried. for line in walker: if line.startswith(''): break newfname = os.path.splitext(infile)[0] + '.html' with open(newfname, 'w') as f: for line in walker: if line.startswith(''): break f.write(line) f.write('\n') return newfname #----------------------------------------------------------------------------- # Cell-level functions -- similar to IPython.nbformat.v3.rwbase functions # but at cell level instead of whole notebook level #----------------------------------------------------------------------------- def writes_cell(cell, **kwargs): kwargs['cls'] = BytesEncoder kwargs['indent'] = 3 kwargs['sort_keys'] = True kwargs['separators'] = (',',': ') if kwargs.pop('split_lines', True): cell = split_lines_cell(copy.deepcopy(cell)) return py3compat.str_to_unicode(json.dumps(cell, **kwargs), 'utf-8') _multiline_outputs = ['text', 'html', 'svg', 'latex', 'javascript', 'json'] def split_lines_cell(cell): """ Split lines within a cell as in IPython.nbformat.v3.rwbase.split_lines """ if cell.cell_type == 'code': if 'input' in cell and isinstance(cell.input, basestring): cell.input = (cell.input + '\n').splitlines() for output in cell.outputs: for key in _multiline_outputs: item = output.get(key, None) if isinstance(item, basestring): output[key] = (item + '\n').splitlines() else: # text, heading cell for key in ['source', 'rendered']: item = cell.get(key, None) if isinstance(item, basestring): cell[key] = (item + '\n').splitlines() return cell def cell_to_lines(cell): ''' Write a cell to json, returning the split lines. ''' split_lines_cell(cell) s = writes_cell(cell).strip() return s.split('\n') known_formats = "rst (default), html, blogger-html, latex, markdown, py" def main(infile, format='rst'): """Convert a notebook to html in one step""" # XXX: this is just quick and dirty for now. When adding a new format, # make sure to add it to the `known_formats` string above, which gets # printed in in the catch-all else, as well as in the help if format == 'rst': converter = ConverterRST(infile) converter.render() elif format == 'markdown': converter = ConverterMarkdown(infile) converter.render() elif format == 'html': converter = ConverterHTML(infile) htmlfname = converter.render() elif format == 'blogger-html': converter = ConverterBloggerHTML(infile) htmlfname = converter.render() elif format == 'latex': converter = ConverterLaTeX(infile) latexfname = converter.render() elif format == 'py': converter = ConverterPy(infile) converter.render() else: raise SystemExit("Unknown format '%s', " % format + "known formats are: " + known_formats) #----------------------------------------------------------------------------- # Script main #----------------------------------------------------------------------------- if __name__ == '__main__': parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawTextHelpFormatter) # TODO: consider passing file like object around, rather than filenames # would allow us to process stdin, or even http streams #parser.add_argument('infile', nargs='?', type=argparse.FileType('r'), default=sys.stdin) #Require a filename as a positional argument parser.add_argument('infile', nargs=1) parser.add_argument('-f', '--format', default='rst', help='Output format. Supported formats: \n' + known_formats) args = parser.parse_args() main(infile=args.infile[0], format=args.format)