more test fixed

converters/base.py

@@ -1,333 +1,334 b''
 from __future__ import print_function
 from converters.utils import remove_fake_files_url
 
 # Stdlib
 import codecs
 import io
 import logging
 import os
 import pprint
 import re
 import subprocess
 import sys
 import json
 import copy
 from types import FunctionType
 from shutil import rmtree
 
 # From IPython
 from IPython.external import argparse
 from IPython.nbformat import current as nbformat
 from IPython.utils.text import indent
 from IPython.nbformat.v3.nbjson import BytesEncoder
 from IPython.utils import path, py3compat
 
 # local
 from lexers import IPythonLexer
 
 #-----------------------------------------------------------------------------
 # Class declarations
 #-----------------------------------------------------------------------------
 
 class ConversionException(Exception):
     pass
 
 class DocStringInheritor(type):
     """
     This metaclass will walk the list of bases until the desired
     superclass method is found AND if that method has a docstring and only
     THEN does it attach the superdocstring to the derived class method.
 
     Please use carefully, I just did the metaclass thing by following
     Michael Foord's Metaclass tutorial
     (http://www.voidspace.org.uk/python/articles/metaclasses.shtml), I may
     have missed a step or two.
 
     source:
     http://groups.google.com/group/comp.lang.python/msg/26f7b4fcb4d66c95
     by Paul McGuire
     """
     def __new__(meta, classname, bases, classDict):
         newClassDict = {}
         for attributeName, attribute in classDict.items():
             if type(attribute) == FunctionType:
                 # look through bases for matching function by name
                 for baseclass in bases:
                     if hasattr(baseclass, attributeName):
                         basefn = getattr(baseclass,attributeName)
                         if basefn.__doc__:
                             attribute.__doc__ = basefn.__doc__
                             break
             newClassDict[attributeName] = attribute
         return type.__new__(meta, classname, bases, newClassDict)
 
 class Converter(object):
     __metaclass__ = DocStringInheritor
     default_encoding = 'utf-8'
     extension = str()
     figures_counter = 0
     infile = str()
     infile_dir = str()
     infile_root = str()
     files_dir = str()
     with_preamble = True
     user_preamble = None
-    output = str()
+    output = unicode()
     raw_as_verbatim = False
     
     def __init__(self, infile):
         self.infile = infile
         self.infile_dir, infile_root = os.path.split(infile)
         infile_root = os.path.splitext(infile_root)[0]
         files_dir = os.path.join(self.infile_dir, infile_root + '_files')
         if not os.path.isdir(files_dir):
             os.mkdir(files_dir)
         self.infile_root = infile_root
         self.files_dir = files_dir
         self.outbase = os.path.join(self.infile_dir, infile_root)
 
     def __del__(self):
         if not os.listdir(self.files_dir):
             os.rmdir(self.files_dir)
 
     def dispatch(self, cell_type):
         """return cell_type dependent render method,  for example render_code
         """
         return getattr(self, 'render_' + cell_type, self.render_unknown)
 
     def dispatch_display_format(self, format):
         """return output_type dependent render method,  for example render_output_text
         """
         return getattr(self, 'render_display_format_' + format, self.render_unknown_display)
 
     def convert(self, cell_separator='\n'):
         """
         Generic method to converts notebook to a string representation.
 
         This is accomplished by dispatching on the cell_type, so subclasses of
         Convereter class do not need to re-implement this method, but just
         need implementation for the methods that will be dispatched.
 
         Parameters
         ----------
         cell_separator : string
           Character or string to join cells with. Default is "\n"
 
         Returns
         -------
         out : string
         """
         lines = []
         lines.extend(self.optional_header())
         lines.extend(self.main_body(cell_separator))
         lines.extend(self.optional_footer())
         return u'\n'.join(lines)
 
     def main_body(self, cell_separator='\n'):
         converted_cells = []
         for worksheet in self.nb.worksheets:
             for cell in worksheet.cells:
                 #print(cell.cell_type)  # dbg
                 conv_fn = self.dispatch(cell.cell_type)
                 if cell.cell_type in ('markdown', 'raw'):
                     remove_fake_files_url(cell)
                 converted_cells.append('\n'.join(conv_fn(cell)))
         cell_lines = cell_separator.join(converted_cells).split('\n')
         return cell_lines
 
     def render(self):
         "read, convert, and save self.infile"
         if not hasattr(self, 'nb'):
             self.read()
         self.output = self.convert()
+        assert(type(self.output) == unicode)
         return self.save()
 
     def read(self):
         "read and parse notebook into NotebookNode called self.nb"
         with open(self.infile) as f:
             self.nb = nbformat.read(f, 'json')
 
     def save(self, outfile=None, encoding=None):
         "read and parse notebook into self.nb"
         if outfile is None:
             outfile = self.outbase + '.' + self.extension
         if encoding is None:
             encoding = self.default_encoding
         with io.open(outfile, 'w', encoding=encoding) as f:
             f.write(self.output)
         return os.path.abspath(outfile)
 
     def optional_header(self):
         """
         Optional header to insert at the top of the converted notebook
 
         Returns a list
         """
         return []
 
     def optional_footer(self):
         """
         Optional footer to insert at the end of the converted notebook
 
         Returns a list
         """
         return []
 
     def _new_figure(self, data, fmt):
         """Create a new figure file in the given format.
 
         Returns a path relative to the input file.
         """
         figname = '%s_fig_%02i.%s' % (self.infile_root, 
                                       self.figures_counter, fmt)
         self.figures_counter += 1
         fullname = os.path.join(self.files_dir, figname)
 
         # Binary files are base64-encoded, SVG is already XML
         if fmt in ('png', 'jpg', 'pdf'):
             data = data.decode('base64')
             fopen = lambda fname: open(fname, 'wb')
         else:
             fopen = lambda fname: codecs.open(fname, 'wb', self.default_encoding)
             
         with fopen(fullname) as f:
             f.write(data)
             
         return fullname
 
     def render_heading(self, cell):
         """convert a heading cell
 
         Returns list."""
         raise NotImplementedError
 
     def render_code(self, cell):
         """Convert a code cell
 
         Returns list."""
         raise NotImplementedError
 
     def render_markdown(self, cell):
         """convert a markdown cell
 
         Returns list."""
         raise NotImplementedError
 
     def _img_lines(self, img_file):
         """Return list of lines to include an image file."""
         # Note: subclasses may choose to implement format-specific _FMT_lines
         # methods if they so choose (FMT in {png, svg, jpg, pdf}).
         raise NotImplementedError
 
     def render_display_data(self, output):
         """convert display data from the output of a code cell
 
         Returns list.
         """
         lines = []
 
         for fmt in output.keys():
             if fmt in ['png', 'svg', 'jpg', 'pdf']:
                 img_file = self._new_figure(output[fmt], fmt)
                 # Subclasses can have format-specific render functions (e.g.,
                 # latex has to auto-convert all SVG to PDF first).
                 lines_fun = getattr(self, '_%s_lines' % fmt, None)
                 if not lines_fun:
                     lines_fun = self._img_lines
                 lines.extend(lines_fun(img_file))
             elif fmt != 'output_type':
                 conv_fn = self.dispatch_display_format(fmt)
                 lines.extend(conv_fn(output))
         return lines
 
     def render_raw(self, cell):
         """convert a cell with raw text
 
         Returns list."""
         raise NotImplementedError
 
     def render_unknown(self, cell):
         """Render cells of unkown type
 
         Returns list."""
         data = pprint.pformat(cell)
         logging.warning('Unknown cell: %s' % cell.cell_type)
         return self._unknown_lines(data)
 
     def render_unknown_display(self, output, type):
         """Render cells of unkown type
 
         Returns list."""
         data = pprint.pformat(output)
         logging.warning('Unknown output: %s' % output.output_type)
         return self._unknown_lines(data)
 
     def render_stream(self, output):
         """render the stream part of an output
 
         Returns list.
 
         Identical to render_display_format_text
         """
         return self.render_display_format_text(output)
 
     def render_pyout(self, output):
         """convert pyout part of a code cell
 
         Returns list."""
         raise NotImplementedError
 
 
     def render_pyerr(self, output):
         """convert pyerr part of a code cell
 
         Returns list."""
         raise NotImplementedError
 
     def _unknown_lines(self, data):
         """Return list of lines for an unknown cell.
 
         Parameters
         ----------
         data : str
           The content of the unknown data as a single string.
         """
         raise NotImplementedError
 
     # These are the possible format types in an output node
 
     def render_display_format_text(self, output):
         """render the text part of an output
 
         Returns list.
         """
         raise NotImplementedError
 
     def render_display_format_html(self, output):
         """render the html part of an output
 
         Returns list.
         """
         raise NotImplementedError
 
     def render_display_format_latex(self, output):
         """render the latex part of an output
 
         Returns list.
         """
         raise NotImplementedError
 
     def render_display_format_json(self, output):
         """render the json part of an output
 
         Returns list.
         """
         raise NotImplementedError
 
     def render_display_format_javascript(self, output):
         """render the javascript part of an output
 
         Returns list.
         """
         raise NotImplementedError
 

converters/notebook.py

@@ -1,81 +1,82 b''
 from converters.base import Converter
+from converters.utils import cell_to_lines
 from shutil import rmtree
 import json
 
 class ConverterNotebook(Converter):
     """
     A converter that is essentially a null-op.
     This exists so it can be subclassed
     for custom handlers of .ipynb files 
     that create new .ipynb files.
 
     What distinguishes this from JSONWriter is that
     subclasses can specify what to do with each type of cell.
 
     Writes out a notebook file.
 
     """
     extension = 'ipynb'
 
     def __init__(self, infile, outbase):
         Converter.__init__(self, infile)
         self.outbase = outbase
         rmtree(self.files_dir)
 
     def convert(self):
-        return json.dumps(json.loads(Converter.convert(self, ',')), indent=1, sort_keys=True)
+        return unicode(json.dumps(json.loads(Converter.convert(self, ',')), indent=1, sort_keys=True))
 
     def optional_header(self):
         s = \
 """{
  "metadata": {
  "name": "%(name)s"
  },
  "nbformat": 3,
  "worksheets": [
  {
  "cells": [""" % {'name':self.outbase}
 
         return s.split('\n')
 
     def optional_footer(self):
         s = \
 """]
   }
  ]
 }"""
         return s.split('\n')
 
     def render_heading(self, cell):
         return cell_to_lines(cell)
 
     def render_code(self, cell):
         return cell_to_lines(cell)
 
     def render_markdown(self, cell):
         return cell_to_lines(cell)
 
     def render_raw(self, cell):
         return cell_to_lines(cell)
 
     def render_pyout(self, output):
         return cell_to_lines(output)
 
     def render_pyerr(self, output):
         return cell_to_lines(output)
 
     def render_display_format_text(self, output):
         return [output.text]
 
     def render_display_format_html(self, output):
         return [output.html]
 
     def render_display_format_latex(self, output):
         return [output.latex]
 
     def render_display_format_json(self, output):
         return [output.json]
 
 
     def render_display_format_javascript(self, output):
         return [output.javascript]

converters/utils.py

@@ -1,327 +1,333 b''
 from __future__ import print_function
 from lexers import IPythonLexer
+
 import subprocess
+import copy
+import json
 import re
+
 from IPython.utils.text import indent
+from IPython.utils import path, py3compat
+from IPython.nbformat.v3.nbjson import BytesEncoder
 
 #-----------------------------------------------------------------------------
 # Utility functions
 #-----------------------------------------------------------------------------
 def highlight(src, lang='ipython'):
     """Return a syntax-highlighted version of the input source.
     """
     from pygments import highlight
     from pygments.lexers import get_lexer_by_name
     from pygments.formatters import HtmlFormatter
     
     if lang == 'ipython':
         lexer = IPythonLexer()
     else:
         lexer = get_lexer_by_name(lang, stripall=True)
         
     return highlight(src, lexer, HtmlFormatter())
 
 def output_container(f):
     """add a prompt-area next to an output"""
     def wrapped(self, output):
         rendered = f(self, output)
         if not rendered:
             # empty output
             return []
         lines = []
         lines.append('<div class="hbox output_area">')
         lines.extend(self._out_prompt(output))
         classes = "output_subarea output_%s" % output.output_type
         if 'html' in output.keys():
             classes +=  ' output_html rendered_html'
         if output.output_type == 'stream':
             classes += " output_%s" % output.stream
         lines.append('<div class="%s">' % classes)
         lines.extend(rendered)
         lines.append('</div>') # subarea
         lines.append('</div>') # output_area
         
         return lines
     
     return wrapped
 
 def text_cell(f):
     """wrap text cells in appropriate divs"""
     def wrapped(self, cell):
         rendered = f(self, cell)
         classes = "text_cell_render border-box-sizing rendered_html"
         lines = ['<div class="%s">' % classes] + rendered + ['</div>']
         return lines
     return wrapped
 
 
 def remove_fake_files_url(cell):
     """Remove from the cell source the /files/ pseudo-path we use.
     """
     src = cell.source
     cell.source = src.replace('/files/', '')
 
 
 # ANSI color functions:
 
 def remove_ansi(src):
     """Strip all ANSI color escape sequences from input string.
 
     Parameters
     ----------
     src : string
 
     Returns
     -------
     string
     """
     return re.sub(r'\033\[(0|\d;\d\d)m', '', src)
 
 
 def ansi2html(txt):
     """Render ANSI colors as HTML colors
     
     This is equivalent to util.fixConsole in utils.js
     
     Parameters
     ----------
     txt : string
 
     Returns
     -------
     string
     """
     
     ansi_colormap = {
         '30': 'ansiblack',
         '31': 'ansired',
         '32': 'ansigreen',
         '33': 'ansiyellow',
         '34': 'ansiblue',
         '35': 'ansipurple',
         '36': 'ansicyan',
         '37': 'ansigrey',
         '01': 'ansibold',
     }
     
     # do ampersand first
     txt = txt.replace('&', '&amp;')
     html_escapes = {
         '<': '&lt;',
         '>': '&gt;',
         "'": '&apos;',
         '"': '&quot;',
         '`': '&#96;',
     }
     for c, escape in html_escapes.iteritems():
         txt = txt.replace(c, escape)
     
     ansi_re = re.compile('\x1b' + r'\[([\dA-Fa-f;]*?)m')
     m = ansi_re.search(txt)
     opened = False
     cmds = []
     opener = ''
     closer = ''
     while m:
         cmds = m.groups()[0].split(';')
         closer = '</span>' if opened else ''
         opened = len(cmds) > 1 or cmds[0] != '0'*len(cmds[0]);
         classes = []
         for cmd in cmds:
             if cmd in ansi_colormap:
                 classes.append(ansi_colormap.get(cmd))
         
         if classes:
             opener = '<span class="%s">' % (' '.join(classes))
         else:
             opener = ''
         txt = re.sub(ansi_re, closer + opener, txt, 1)
         
         m = ansi_re.search(txt)
     
     if opened:
         txt += '</span>'
     return txt
 
 
 # Pandoc-dependent code
 
 def markdown2latex(src):
     """Convert a markdown string to LaTeX via pandoc.
 
     This function will raise an error if pandoc is not installed.
 
     Any error messages generated by pandoc are printed to stderr.
 
     Parameters
     ----------
     src : string
       Input string, assumed to be valid markdown.
 
     Returns
     -------
     out : string
       Output as returned by pandoc.
     """
     p = subprocess.Popen('pandoc -f markdown -t latex'.split(),
                          stdin=subprocess.PIPE, stdout=subprocess.PIPE)
     out, err = p.communicate(src.encode('utf-8'))
     if err:
         print(err, file=sys.stderr)
     #print('*'*20+'\n', out, '\n'+'*'*20)  # dbg
     return unicode(out,'utf-8')
 
 
 def markdown2rst(src):
     """Convert a markdown string to LaTeX via pandoc.
 
     This function will raise an error if pandoc is not installed.
 
     Any error messages generated by pandoc are printed to stderr.
 
     Parameters
     ----------
     src : string
       Input string, assumed to be valid markdown.
 
     Returns
     -------
     out : string
       Output as returned by pandoc.
     """
     p = subprocess.Popen('pandoc -f markdown -t rst'.split(),
                          stdin=subprocess.PIPE, stdout=subprocess.PIPE)
     out, err = p.communicate(src.encode('utf-8'))
     if err:
         print(err, file=sys.stderr)
     #print('*'*20+'\n', out, '\n'+'*'*20)  # dbg
     return unicode(out,'utf-8')
 
 
 def rst_directive(directive, text=''):
     """
     Makes ReST directive block and indents any text passed to it.
     """
     out = [directive, '']
     if text:
         out.extend([indent(text), ''])
     return out
 
 
 def coalesce_streams(outputs):
     """merge consecutive sequences of stream output into single stream
     
     to prevent extra newlines inserted at flush calls
     
     TODO: handle \r deletion
     """
     new_outputs = []
     last = outputs[0]
     new_outputs = [last]
     for output in outputs[1:]:
         if (output.output_type == 'stream' and
             last.output_type == 'stream' and
             last.stream == output.stream
         ):
             last.text += output.text
         else:
             new_outputs.append(output)
     
     return new_outputs
 
 
 def rst2simplehtml(infile):
     """Convert a rst file to simplified html suitable for blogger.
 
     This just runs rst2html with certain parameters to produce really simple
     html and strips the document header, so the resulting file can be easily
     pasted into a blogger edit window.
     """
 
     # This is the template for the rst2html call that produces the cleanest,
     # simplest html I could find.  This should help in making it easier to
     # paste into the blogspot html window, though I'm still having problems
     # with linebreaks there...
     cmd_template = ("rst2html --link-stylesheet --no-xml-declaration "
                     "--no-generator --no-datestamp --no-source-link "
                     "--no-toc-backlinks --no-section-numbering "
                     "--strip-comments ")
 
     cmd = "%s %s" % (cmd_template, infile)
     proc = subprocess.Popen(cmd,
                             stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE,
                             shell=True)
     html, stderr = proc.communicate()
     if stderr:
         raise IOError(stderr)
 
     # Make an iterator so breaking out holds state.  Our implementation of
     # searching for the html body below is basically a trivial little state
     # machine, so we need this.
     walker = iter(html.splitlines())
 
     # Find start of main text, break out to then print until we find end /div.
     # This may only work if there's a real title defined so we get a 'div class'
     # tag, I haven't really tried.
     for line in walker:
         if line.startswith('<body>'):
             break
 
     newfname = os.path.splitext(infile)[0] + '.html'
     with open(newfname, 'w') as f:
         for line in walker:
             if line.startswith('</body>'):
                 break
             f.write(line)
             f.write('\n')
 
     return newfname
 
 #-----------------------------------------------------------------------------
 # Cell-level functions -- similar to IPython.nbformat.v3.rwbase functions
 # but at cell level instead of whole notebook level
 #-----------------------------------------------------------------------------
 
 def writes_cell(cell, **kwargs):
     kwargs['cls'] = BytesEncoder
     kwargs['indent'] = 3
     kwargs['sort_keys'] = True
     kwargs['separators'] = (',',': ')
     if kwargs.pop('split_lines', True):
         cell = split_lines_cell(copy.deepcopy(cell))
     return py3compat.str_to_unicode(json.dumps(cell, **kwargs), 'utf-8')
 
 
 _multiline_outputs = ['text', 'html', 'svg', 'latex', 'javascript', 'json']
 
 
 def split_lines_cell(cell):
     """
     Split lines within a cell as in 
     IPython.nbformat.v3.rwbase.split_lines
 
     """
     if cell.cell_type == 'code':
         if 'input' in cell and isinstance(cell.input, basestring):
             cell.input = (cell.input + '\n').splitlines()
         for output in cell.outputs:
             for key in _multiline_outputs:
                 item = output.get(key, None)
                 if isinstance(item, basestring):
                     output[key] = (item + '\n').splitlines()
     else: # text, heading cell
         for key in ['source', 'rendered']:
             item = cell.get(key, None)
             if isinstance(item, basestring):
                 cell[key] = (item + '\n').splitlines()
     return cell
 
 
 def cell_to_lines(cell):
     '''
     Write a cell to json, returning the split lines.
     '''
     split_lines_cell(cell)
     s = writes_cell(cell).strip()
     return s.split('\n')