transformers.py
269 lines
| 8.7 KiB
| text/x-python
|
PythonLexer
/ converters / transformers.py
Matthias BUSSONNIER
|
r9302 | """ | ||
Matthias BUSSONNIER
|
r9528 | Module that regroups transformer that woudl be applied to ipynb files | ||
before going through the templating machinery. | ||||
Matthias BUSSONNIER
|
r9302 | |||
Matthias BUSSONNIER
|
r9528 | It exposes convenient classes to inherit from to access configurability | ||
as well as decorator to simplify tasks. | ||||
Matthias BUSSONNIER
|
r9302 | """ | ||
Matthias BUSSONNIER
|
r9490 | from __future__ import print_function | ||
Matthias BUSSONNIER
|
r9307 | from IPython.config.configurable import Configurable | ||
Matthias BUSSONNIER
|
r9312 | from IPython.utils.traitlets import Unicode, Bool, Dict, List | ||
Matthias BUSSONNIER
|
r9307 | |||
Matthias BUSSONNIER
|
r9529 | from converters.config import GlobalConfigurable | ||
class ConfigurableTransformers(GlobalConfigurable): | ||||
Matthias BUSSONNIER
|
r9528 | """ A configurable transformer | ||
Inherit from this class if you wish to have configurability for your | ||||
transformer. | ||||
Any configurable traitlets this class exposed will be configurable in profiles | ||||
using c.SubClassName.atribute=value | ||||
you can overwrite cell_transform to apply a transformation independently on each cell | ||||
or __call__ if you prefer your own logic. See orresponding docstring for informations. | ||||
""" | ||||
Matthias BUSSONNIER
|
r9307 | |||
def __init__(self, config=None, **kw): | ||||
super(ConfigurableTransformers, self).__init__(config=config, **kw) | ||||
def __call__(self, nb, other): | ||||
Matthias BUSSONNIER
|
r9528 | """transformation to apply on each notebook. | ||
received a handle to the current notebook as well as a dict of resources | ||||
which structure depends on the transformer. | ||||
You should return modified nb, other. | ||||
If you wish to apply on each cell, you might want to overwrite cell_transform method. | ||||
""" | ||||
Matthias BUSSONNIER
|
r9307 | try : | ||
for worksheet in nb.worksheets : | ||||
for index, cell in enumerate(worksheet.cells): | ||||
worksheet.cells[index], other = self.cell_transform(cell, other, index) | ||||
return nb, other | ||||
Matthias BUSSONNIER
|
r9491 | except NotImplementedError: | ||
Matthias BUSSONNIER
|
r9307 | raise NotImplementedError('should be implemented by subclass') | ||
def cell_transform(self, cell, other, index): | ||||
""" | ||||
Matthias BUSSONNIER
|
r9528 | Overwrite if you want to apply a transformation on each cell, | ||
receive the current cell, the resource dict and the index of current cell as parameter. | ||||
You should return modified cell and resource dict. | ||||
Matthias BUSSONNIER
|
r9307 | """ | ||
raise NotImplementedError('should be implemented by subclass') | ||||
Matthias BUSSONNIER
|
r9528 | return cell, other | ||
Matthias BUSSONNIER
|
r9307 | |||
Matthias BUSSONNIER
|
r9420 | class ActivatableTransformer(ConfigurableTransformers): | ||
Matthias BUSSONNIER
|
r9528 | """A simple ConfigurableTransformers that have an enabled flag | ||
Inherit from that if you just want to have a transformer which is | ||||
no-op by default but can be activated in profiles with | ||||
c.YourTransformerName.enabled = True | ||||
""" | ||||
Matthias BUSSONNIER
|
r9307 | |||
Matthias BUSSONNIER
|
r9421 | enabled = Bool(False, config=True) | ||
Matthias BUSSONNIER
|
r9307 | |||
Matthias BUSSONNIER
|
r9420 | def __call__(self, nb, other): | ||
Matthias BUSSONNIER
|
r9421 | if not self.enabled : | ||
Matthias BUSSONNIER
|
r9491 | return nb, other | ||
Matthias BUSSONNIER
|
r9420 | else : | ||
Matthias BUSSONNIER
|
r9491 | return super(ActivatableTransformer, self).__call__(nb, other) | ||
Matthias BUSSONNIER
|
r9307 | |||
Matthias BUSSONNIER
|
r9302 | def cell_preprocessor(function): | ||
""" wrap a function to be executed on all cells of a notebook | ||||
wrapped function parameters : | ||||
Matthias BUSSONNIER
|
r9528 | cell : the cell | ||
other : external resources | ||||
index : index of the cell | ||||
Matthias BUSSONNIER
|
r9302 | """ | ||
Matthias BUSSONNIER
|
r9303 | def wrappedfunc(nb, other): | ||
Matthias BUSSONNIER
|
r9302 | for worksheet in nb.worksheets : | ||
for index, cell in enumerate(worksheet.cells): | ||||
Matthias BUSSONNIER
|
r9303 | worksheet.cells[index], other = function(cell, other, index) | ||
return nb, other | ||||
Matthias BUSSONNIER
|
r9302 | return wrappedfunc | ||
@cell_preprocessor | ||||
def haspyout_transformer(cell, other, count): | ||||
""" | ||||
Add a haspyout flag to cell that have it | ||||
Matthias BUSSONNIER
|
r9307 | |||
Matthias BUSSONNIER
|
r9302 | Easier for templating, where you can't know in advance | ||
wether to write the out prompt | ||||
""" | ||||
cell.type = cell.cell_type | ||||
cell.haspyout = False | ||||
for out in cell.get('outputs', []): | ||||
if out.output_type == 'pyout': | ||||
cell.haspyout = True | ||||
break | ||||
Matthias BUSSONNIER
|
r9303 | return cell, other | ||
Matthias BUSSONNIER
|
r9302 | |||
Matthias BUSSONNIER
|
r9492 | @cell_preprocessor | ||
def coalesce_streams(cell, other, count): | ||||
"""merge consecutive sequences of stream output into single stream | ||||
to prevent extra newlines inserted at flush calls | ||||
TODO: handle \r deletion | ||||
""" | ||||
Matthias BUSSONNIER
|
r9494 | outputs = cell.get('outputs', []) | ||
Matthias BUSSONNIER
|
r9492 | if not outputs: | ||
return cell, other | ||||
new_outputs = [] | ||||
last = outputs[0] | ||||
new_outputs = [last] | ||||
for output in outputs[1:]: | ||||
if (output.output_type == 'stream' and | ||||
last.output_type == 'stream' and | ||||
last.stream == output.stream | ||||
): | ||||
last.text += output.text | ||||
else: | ||||
new_outputs.append(output) | ||||
cell.outputs = new_outputs | ||||
return cell, other | ||||
Matthias BUSSONNIER
|
r9420 | class ExtractFigureTransformer(ActivatableTransformer): | ||
Matthias BUSSONNIER
|
r9303 | |||
Matthias BUSSONNIER
|
r9528 | |||
Matthias BUSSONNIER
|
r9307 | extra_ext_map = Dict({}, | ||
config=True, | ||||
help="""extra map to override extension based on type. | ||||
Usefull for latex where svg will be converted to pdf before inclusion | ||||
""" | ||||
) | ||||
Matthias BUSSONNIER
|
r9490 | |||
Matthias BUSSONNIER
|
r9493 | key_format_map = Dict({}, | ||
config=True, | ||||
) | ||||
Matthias BUSSONNIER
|
r9494 | figname_format_map = Dict({}, | ||
config=True, | ||||
) | ||||
Matthias BUSSONNIER
|
r9303 | |||
Matthias BUSSONNIER
|
r9528 | |||
Matthias BUSSONNIER
|
r9307 | #to do change this to .format {} syntax | ||
Matthias BUSSONNIER
|
r9494 | default_key_tpl = Unicode('_fig_{count:02d}.{ext}', config=True) | ||
Matthias BUSSONNIER
|
r9307 | |||
def _get_ext(self, ext): | ||||
if ext in self.extra_ext_map : | ||||
return self.extra_ext_map[ext] | ||||
return ext | ||||
def _new_figure(self, data, fmt, count): | ||||
"""Create a new figure file in the given format. | ||||
""" | ||||
Matthias BUSSONNIER
|
r9530 | tplf = self.figname_format_map.get(fmt, self.default_key_tpl) | ||
tplk = self.key_format_map.get(fmt, self.default_key_tpl) | ||||
Matthias BUSSONNIER
|
r9494 | # option to pass the hash as data ? | ||
figname = tplf.format(count=count, ext=self._get_ext(fmt)) | ||||
key = tplk.format(count=count, ext=self._get_ext(fmt)) | ||||
Matthias BUSSONNIER
|
r9307 | |||
# Binary files are base64-encoded, SVG is already XML | ||||
if fmt in ('png', 'jpg', 'pdf'): | ||||
data = data.decode('base64') | ||||
return figname, key, data | ||||
def cell_transform(self, cell, other, count): | ||||
Matthias BUSSONNIER
|
r9491 | if other.get('figures', None) is None : | ||
other['figures'] = {} | ||||
for out in cell.get('outputs', []): | ||||
for out_type in self.display_data_priority: | ||||
if out.hasattr(out_type): | ||||
figname, key, data = self._new_figure(out[out_type], out_type, count) | ||||
out['key_'+out_type] = figname | ||||
Matthias BUSSONNIER
|
r9331 | other['figures'][key] = data | ||
Matthias BUSSONNIER
|
r9307 | count = count+1 | ||
return cell, other | ||||
Matthias BUSSONNIER
|
r9302 | |||
Matthias BUSSONNIER
|
r9401 | |||
class RevealHelpTransformer(ConfigurableTransformers): | ||||
damianavila
|
r9509 | def __call__(self, nb, other): | ||
damianavila
|
r9511 | for worksheet in nb.worksheets : | ||
for i, cell in enumerate(worksheet.cells): | ||||
cell.metadata.slide_type = cell.metadata.get('slideshow', {}).get('slide_type', None) | ||||
if cell.metadata.slide_type is None: | ||||
cell.metadata.slide_type = '-' | ||||
if cell.metadata.slide_type in ['slide']: | ||||
worksheet.cells[i - 1].metadata.slide_helper = 'slide_end' | ||||
if cell.metadata.slide_type in ['subslide']: | ||||
worksheet.cells[i - 1].metadata.slide_helper = 'subslide_end' | ||||
return nb, other | ||||
Matthias BUSSONNIER
|
r9401 | |||
Matthias BUSSONNIER
|
r9490 | |||
class CSSHtmlHeaderTransformer(ActivatableTransformer): | ||||
Matthias BUSSONNIER
|
r9492 | |||
Matthias BUSSONNIER
|
r9490 | def __call__(self, nb, resources): | ||
"""Fetch and add css to the resource dict | ||||
Matthias BUSSONNIER
|
r9492 | Fetch css from IPython adn Pygment to add at the beginning | ||
of the html files. | ||||
Matthias BUSSONNIER
|
r9490 | |||
Add this css in resources in the "inlining.css" key | ||||
""" | ||||
Matthias BUSSONNIER
|
r9491 | resources['inlining'] = {} | ||
Matthias BUSSONNIER
|
r9490 | resources['inlining']['css'] = self.header | ||
return nb, resources | ||||
Matthias BUSSONNIER
|
r9491 | header = [] | ||
Matthias BUSSONNIER
|
r9490 | |||
def __init__(self, config=None, **kw): | ||||
Matthias BUSSONNIER
|
r9491 | super(CSSHtmlHeaderTransformer, self).__init__(config=config, **kw) | ||
Matthias BUSSONNIER
|
r9490 | if self.enabled : | ||
self.regen_header() | ||||
def regen_header(self): | ||||
## lazy load asa this might not be use in many transformers | ||||
import os | ||||
from IPython.utils import path | ||||
import io | ||||
from pygments.formatters import HtmlFormatter | ||||
header = [] | ||||
static = os.path.join(path.get_ipython_package_dir(), | ||||
'frontend', 'html', 'notebook', 'static', | ||||
) | ||||
here = os.path.split(os.path.realpath(__file__))[0] | ||||
css = os.path.join(static, 'css') | ||||
for sheet in [ | ||||
# do we need jquery and prettify? | ||||
# os.path.join(static, 'jquery', 'css', 'themes', 'base', | ||||
# 'jquery-ui.min.css'), | ||||
# os.path.join(static, 'prettify', 'prettify.css'), | ||||
os.path.join(css, 'boilerplate.css'), | ||||
os.path.join(css, 'fbm.css'), | ||||
os.path.join(css, 'notebook.css'), | ||||
os.path.join(css, 'renderedhtml.css'), | ||||
os.path.join(css, 'style.min.css'), | ||||
]: | ||||
try: | ||||
with io.open(sheet, encoding='utf-8') as f: | ||||
s = f.read() | ||||
header.append(s) | ||||
except IOError: | ||||
# new version of ipython with style.min.css, pass | ||||
pass | ||||
pygments_css = HtmlFormatter().get_style_defs('.highlight') | ||||
header.append(pygments_css) | ||||
self.header = header | ||||