##// END OF EJS Templates
move transformer in separate file
Matthias BUSSONNIER -
Show More
@@ -0,0 +1,50 b''
1 """
2
3 """
4
5 def cell_preprocessor(function):
6 """ wrap a function to be executed on all cells of a notebook
7
8 wrapped function parameters :
9 cell : the cell
10 other : external resources
11 index : index of the cell
12 """
13 def wrappedfunc(nb,other):
14 for worksheet in nb.worksheets :
15 for index, cell in enumerate(worksheet.cells):
16 worksheet.cells[index],other= function(cell,other,index)
17 return nb,other
18 return wrappedfunc
19
20
21 @cell_preprocessor
22 def haspyout_transformer(cell, other, count):
23 """
24 Add a haspyout flag to cell that have it
25
26 Easier for templating, where you can't know in advance
27 wether to write the out prompt
28
29 """
30 cell.type = cell.cell_type
31 cell.haspyout = False
32 for out in cell.get('outputs', []):
33 if out.output_type == 'pyout':
34 cell.haspyout = True
35 break
36 return cell,other
37
38
39 @cell_preprocessor
40 def extract_figure_transformer(cell,other,count):
41 for i,out in enumerate(cell.get('outputs', [])):
42 for type in ['html', 'pdf', 'svg', 'latex', 'png', 'jpg', 'jpeg']:
43 if out.hasattr(type):
44 figname,data = _new_figure(out[type], type, count)
45 cell.outputs[i][type] = figname
46 out['key_'+type] = figname
47 other[figname] = data
48 count = count+1
49 return cell,other
50
@@ -1,301 +1,274 b''
1 """Base classes for the notebook conversion pipeline.
1 """Base classes for the notebook conversion pipeline.
2
2
3 This module defines Converter, from which all objects designed to implement
3 This module defines Converter, from which all objects designed to implement
4 a conversion of IPython notebooks to some other format should inherit.
4 a conversion of IPython notebooks to some other format should inherit.
5 """
5 """
6 #-----------------------------------------------------------------------------
6 #-----------------------------------------------------------------------------
7 # Copyright (c) 2012, the IPython Development Team.
7 # Copyright (c) 2012, the IPython Development Team.
8 #
8 #
9 # Distributed under the terms of the Modified BSD License.
9 # Distributed under the terms of the Modified BSD License.
10 #
10 #
11 # The full license is in the file COPYING.txt, distributed with this software.
11 # The full license is in the file COPYING.txt, distributed with this software.
12 #-----------------------------------------------------------------------------
12 #-----------------------------------------------------------------------------
13
13
14 #-----------------------------------------------------------------------------
14 #-----------------------------------------------------------------------------
15 # Imports
15 # Imports
16 #-----------------------------------------------------------------------------
16 #-----------------------------------------------------------------------------
17
17
18 from __future__ import print_function, absolute_import
18 from __future__ import print_function, absolute_import
19 from .transformers import extract_figure_transformer
20 import converters.transformers as trans
21
19
22
20 # Stdlib imports
23 # Stdlib imports
21 import io
24 import io
22 import os
25 import os
23 import re
26 import re
24 from IPython.utils import path
27 from IPython.utils import path
25
28
26 from jinja2 import Environment, FileSystemLoader
29 from jinja2 import Environment, FileSystemLoader
27 env = Environment(
30 env = Environment(
28 loader=FileSystemLoader([
31 loader=FileSystemLoader([
29 './templates/',
32 './templates/',
30 './templates/skeleton/',
33 './templates/skeleton/',
31 ]),
34 ]),
32 extensions=['jinja2.ext.loopcontrols']
35 extensions=['jinja2.ext.loopcontrols']
33 )
36 )
34
37
35 texenv = Environment(
38 texenv = Environment(
36 loader=FileSystemLoader([
39 loader=FileSystemLoader([
37 './templates/tex/',
40 './templates/tex/',
38 './templates/skeleton/tex/',
41 './templates/skeleton/tex/',
39 ]),
42 ]),
40 extensions=['jinja2.ext.loopcontrols']
43 extensions=['jinja2.ext.loopcontrols']
41 )
44 )
42
45
43 # IPython imports
46 # IPython imports
44 from IPython.nbformat import current as nbformat
47 from IPython.nbformat import current as nbformat
45 from IPython.config.configurable import Configurable
48 from IPython.config.configurable import Configurable
46 from IPython.utils.traitlets import ( Unicode, Any, List, Bool)
49 from IPython.utils.traitlets import ( Unicode, Any, List, Bool)
47
50
48 # Our own imports
51 # Our own imports
49 from IPython.utils.text import indent
52 from IPython.utils.text import indent
50 from .utils import remove_ansi
53 from .utils import remove_ansi
51 from markdown import markdown
54 from markdown import markdown
52 from .utils import highlight, ansi2html
55 from .utils import highlight, ansi2html
53 from .utils import markdown2latex
56 from .utils import markdown2latex
54 #-----------------------------------------------------------------------------
57 #-----------------------------------------------------------------------------
55 # Class declarations
58 # Class declarations
56 #-----------------------------------------------------------------------------
59 #-----------------------------------------------------------------------------
57 def rm_fake(strng):
60 def rm_fake(strng):
58 return strng.replace('/files/', '')
61 return strng.replace('/files/', '')
59
62
60 class ConversionException(Exception):
63 class ConversionException(Exception):
61 pass
64 pass
62
65
63
66
64 def python_comment(string):
67 def python_comment(string):
65 return '# '+'\n# '.join(string.split('\n'))
68 return '# '+'\n# '.join(string.split('\n'))
66
69
67
70
68
71
69 def header_body():
72 def header_body():
70 """Return the body of the header as a list of strings."""
73 """Return the body of the header as a list of strings."""
71
74
72 from pygments.formatters import HtmlFormatter
75 from pygments.formatters import HtmlFormatter
73
76
74 header = []
77 header = []
75 static = os.path.join(path.get_ipython_package_dir(),
78 static = os.path.join(path.get_ipython_package_dir(),
76 'frontend', 'html', 'notebook', 'static',
79 'frontend', 'html', 'notebook', 'static',
77 )
80 )
78 here = os.path.split(os.path.realpath(__file__))[0]
81 here = os.path.split(os.path.realpath(__file__))[0]
79 css = os.path.join(static, 'css')
82 css = os.path.join(static, 'css')
80 for sheet in [
83 for sheet in [
81 # do we need jquery and prettify?
84 # do we need jquery and prettify?
82 # os.path.join(static, 'jquery', 'css', 'themes', 'base',
85 # os.path.join(static, 'jquery', 'css', 'themes', 'base',
83 # 'jquery-ui.min.css'),
86 # 'jquery-ui.min.css'),
84 # os.path.join(static, 'prettify', 'prettify.css'),
87 # os.path.join(static, 'prettify', 'prettify.css'),
85 os.path.join(css, 'boilerplate.css'),
88 os.path.join(css, 'boilerplate.css'),
86 os.path.join(css, 'fbm.css'),
89 os.path.join(css, 'fbm.css'),
87 os.path.join(css, 'notebook.css'),
90 os.path.join(css, 'notebook.css'),
88 os.path.join(css, 'renderedhtml.css'),
91 os.path.join(css, 'renderedhtml.css'),
89 # our overrides:
92 # our overrides:
90 os.path.join(here, '..', 'css', 'static_html.css'),
93 os.path.join(here, '..', 'css', 'static_html.css'),
91 ]:
94 ]:
92
95
93 with io.open(sheet, encoding='utf-8') as f:
96 with io.open(sheet, encoding='utf-8') as f:
94 s = f.read()
97 s = f.read()
95 header.append(s)
98 header.append(s)
96
99
97 pygments_css = HtmlFormatter().get_style_defs('.highlight')
100 pygments_css = HtmlFormatter().get_style_defs('.highlight')
98 header.append(pygments_css)
101 header.append(pygments_css)
99 return header
102 return header
100
103
101 # todo, make the key part configurable.
104 # todo, make the key part configurable.
102 def _new_figure(data, fmt, count):
105 def _new_figure(data, fmt, count):
103 """Create a new figure file in the given format.
106 """Create a new figure file in the given format.
104
107
105 Returns a path relative to the input file.
108 Returns a path relative to the input file.
106 """
109 """
107 figname = '_fig_%02i.%s' % (count, fmt)
110 figname = '_fig_%02i.%s' % (count, fmt)
108
111
109 # Binary files are base64-encoded, SVG is already XML
112 # Binary files are base64-encoded, SVG is already XML
110 if fmt in ('png', 'jpg', 'pdf'):
113 if fmt in ('png', 'jpg', 'pdf'):
111 data = data.decode('base64')
114 data = data.decode('base64')
112
115
113 return figname,data
116 return figname,data
114
117
115
118
116
119
117
120
118 inlining = {}
121 inlining = {}
119 inlining['css'] = header_body()
122 inlining['css'] = header_body()
120
123
121 LATEX_SUBS = (
124 LATEX_SUBS = (
122 (re.compile(r'\\'), r'\\textbackslash'),
125 (re.compile(r'\\'), r'\\textbackslash'),
123 (re.compile(r'([{}_#%&$])'), r'\\\1'),
126 (re.compile(r'([{}_#%&$])'), r'\\\1'),
124 (re.compile(r'~'), r'\~{}'),
127 (re.compile(r'~'), r'\~{}'),
125 (re.compile(r'\^'), r'\^{}'),
128 (re.compile(r'\^'), r'\^{}'),
126 (re.compile(r'"'), r"''"),
129 (re.compile(r'"'), r"''"),
127 (re.compile(r'\.\.\.+'), r'\\ldots'),
130 (re.compile(r'\.\.\.+'), r'\\ldots'),
128 )
131 )
129
132
130 def escape_tex(value):
133 def escape_tex(value):
131 newval = value
134 newval = value
132 for pattern, replacement in LATEX_SUBS:
135 for pattern, replacement in LATEX_SUBS:
133 newval = pattern.sub(replacement, newval)
136 newval = pattern.sub(replacement, newval)
134 return newval
137 return newval
135
138
136 texenv.block_start_string = '((*'
139 texenv.block_start_string = '((*'
137 texenv.block_end_string = '*))'
140 texenv.block_end_string = '*))'
138 texenv.variable_start_string = '((('
141 texenv.variable_start_string = '((('
139 texenv.variable_end_string = ')))'
142 texenv.variable_end_string = ')))'
140 texenv.comment_start_string = '((='
143 texenv.comment_start_string = '((='
141 texenv.comment_end_string = '=))'
144 texenv.comment_end_string = '=))'
142 texenv.filters['escape_tex'] = escape_tex
145 texenv.filters['escape_tex'] = escape_tex
143
146
144 def cell_preprocessor(function):
147 def cell_preprocessor(function):
145 """ wrap a function to be executed on all cells of a notebook
148 """ wrap a function to be executed on all cells of a notebook
146
149
147 wrapped function parameters :
150 wrapped function parameters :
148 cell : the cell
151 cell : the cell
149 other : external resources
152 other : external resources
150 index : index of the cell
153 index : index of the cell
151 """
154 """
152 def wrappedfunc(nb,other):
155 def wrappedfunc(nb,other):
153 for worksheet in nb.worksheets :
156 for worksheet in nb.worksheets :
154 for index, cell in enumerate(worksheet.cells):
157 for index, cell in enumerate(worksheet.cells):
155 worksheet.cells[index],other= function(cell,other,index)
158 worksheet.cells[index],other= function(cell,other,index)
156 return nb,other
159 return nb,other
157 return wrappedfunc
160 return wrappedfunc
158
161
159
162
160
163
161 @cell_preprocessor
162 def haspyout_transformer(cell, other, count):
163 """
164 Add a haspyout flag to cell that have it
165
166 Easier for templating, where you can't know in advance
167 wether to write the out prompt
168
169 """
170 cell.type = cell.cell_type
171 cell.haspyout = False
172 for out in cell.get('outputs', []):
173 if out.output_type == 'pyout':
174 cell.haspyout = True
175 break
176 return cell,other
177
178
179 @cell_preprocessor
180 def extract_figure_transformer(cell,other,count):
181 for i,out in enumerate(cell.get('outputs', [])):
182 for type in ['html', 'pdf', 'svg', 'latex', 'png', 'jpg', 'jpeg']:
183 if out.hasattr(type):
184 figname,data = _new_figure(out[type], type,count)
185 cell.outputs[i][type] = figname
186 out['key_'+type] = figname
187 other[figname] = data
188 count = count+1
189 return cell,other
190
191
164
192 class ConverterTemplate(Configurable):
165 class ConverterTemplate(Configurable):
193 """ A Jinja2 base converter templates"""
166 """ A Jinja2 base converter templates"""
194
167
195 display_data_priority = List(['html', 'pdf', 'svg', 'latex', 'png', 'jpg', 'jpeg' , 'text'],
168 display_data_priority = List(['html', 'pdf', 'svg', 'latex', 'png', 'jpg', 'jpeg' , 'text'],
196 config=True,
169 config=True,
197 help= """
170 help= """
198 An ordered list of prefered output type, the firs encounterd will usually be
171 An ordered list of prefered output type, the firs encounterd will usually be
199 used when converting discarding the others.
172 used when converting discarding the others.
200 """
173 """
201 )
174 )
202
175
203 pre_transformer_order = List([],
176 pre_transformer_order = List(['haspyout_transformer'],
204 config=True,
177 config=True,
205 help= """ An ordered list of pretransformer to apply to the ipynb file befor running through templates
178 help= """ An ordered list of pretransformer to apply to the ipynb file befor running through templates
206 """
179 """
207 )
180 )
208
181
209 extract_figures = Bool(False,
182 extract_figures = Bool(False,
210 config=True,
183 config=True,
211 help= """
184 help= """
212 wether to remove figure data from ipynb and store them in auxiliary
185 wether to remove figure data from ipynb and store them in auxiliary
213 dictionnary
186 dictionnary
214 """
187 """
215 )
188 )
216
189
217 tex_environement = Bool(False,
190 tex_environement = Bool(False,
218 config=True,
191 config=True,
219 help=""" is this a tex environment or not """)
192 help=""" is this a tex environment or not """)
220
193
221 template_file = Unicode('',
194 template_file = Unicode('',
222 config=True,
195 config=True,
223 help=""" whetever """ )
196 help=""" whetever """ )
224 #-------------------------------------------------------------------------
197 #-------------------------------------------------------------------------
225 # Instance-level attributes that are set in the constructor for this
198 # Instance-level attributes that are set in the constructor for this
226 # class.
199 # class.
227 #-------------------------------------------------------------------------
200 #-------------------------------------------------------------------------
228 infile = Any()
201 infile = Any()
229
202
230
203
231 infile_dir = Unicode()
204 infile_dir = Unicode()
232
205
233 def filter_data_type(self,output):
206 def filter_data_type(self,output):
234 for fmt in self.display_data_priority:
207 for fmt in self.display_data_priority:
235 if fmt in output:
208 if fmt in output:
236 return [fmt]
209 return [fmt]
237
210
238 def __init__(self, preprocessors=[], config=None, **kw):
211 def __init__(self, preprocessors=[], config=None, **kw):
239 """
212 """
240 tplfile : jinja template file to process.
213 tplfile : jinja template file to process.
241
214
242 config: the Configurable confg object to pass around
215 config: the Configurable confg object to pass around
243
216
244 preprocessors: list of function to run on ipynb json data before conversion
217 preprocessors: list of function to run on ipynb json data before conversion
245 to extract/inline file,
218 to extract/inline file,
246
219
247 """
220 """
248 super(ConverterTemplate, self).__init__(config=config, **kw)
221 super(ConverterTemplate, self).__init__(config=config, **kw)
249 self.env = texenv if self.tex_environement else env
222 self.env = texenv if self.tex_environement else env
250 self.ext = '.tplx' if self.tex_environement else '.tpl'
223 self.ext = '.tplx' if self.tex_environement else '.tpl'
251 self.nb = None
224 self.nb = None
252 self.preprocessors = preprocessors
225 self.preprocessors = preprocessors
253 self.preprocessors.append(haspyout_transformer)
226 for name in self.pre_transformer_order:
227 self.preprocessors.append(getattr(trans,'haspyout_transformer'))
254 if self.extract_figures:
228 if self.extract_figures:
255 self.preprocessors.append(extract_figure_transformer)
229 self.preprocessors.append(extract_figure_transformer)
256
230
257 self.env.filters['filter_data_type'] = self.filter_data_type
231 self.env.filters['filter_data_type'] = self.filter_data_type
258 self.env.filters['pycomment'] = python_comment
232 self.env.filters['pycomment'] = python_comment
259 self.env.filters['indent'] = indent
233 self.env.filters['indent'] = indent
260 self.env.filters['rm_fake'] = rm_fake
234 self.env.filters['rm_fake'] = rm_fake
261 self.env.filters['rm_ansi'] = remove_ansi
235 self.env.filters['rm_ansi'] = remove_ansi
262 self.env.filters['markdown'] = markdown
236 self.env.filters['markdown'] = markdown
263 self.env.filters['highlight'] = highlight
237 self.env.filters['highlight'] = highlight
264 self.env.filters['ansi2html'] = ansi2html
238 self.env.filters['ansi2html'] = ansi2html
265 self.env.filters['markdown2latex'] = markdown2latex
239 self.env.filters['markdown2latex'] = markdown2latex
266
240
267 self.template = self.env.get_template(self.template_file+self.ext)
241 self.template = self.env.get_template(self.template_file+self.ext)
268
242
269
243
270
271 def process(self):
244 def process(self):
272 """
245 """
273 preprocess the notebook json for easier use with the templates.
246 preprocess the notebook json for easier use with the templates.
274 will call all the `preprocessor`s in order before returning it.
247 will call all the `preprocessor`s in order before returning it.
275 """
248 """
276 nb = self.nb
249 nb = self.nb
277
250
278 # dict of 'resources' that could be made by the preprocessors
251 # dict of 'resources' that could be made by the preprocessors
279 # like key/value data to extract files from ipynb like in latex conversion
252 # like key/value data to extract files from ipynb like in latex conversion
280 resources = {}
253 resources = {}
281
254
282 for preprocessor in self.preprocessors:
255 for preprocessor in self.preprocessors:
283 nb,resources = preprocessor(nb,resources)
256 nb,resources = preprocessor(nb,resources)
284
257
285 return nb, resources
258 return nb, resources
286
259
287 def convert(self):
260 def convert(self):
288 """ convert the ipynb file
261 """ convert the ipynb file
289
262
290 return both the converted ipynb file and a dict containing potential
263 return both the converted ipynb file and a dict containing potential
291 other resources
264 other resources
292 """
265 """
293 nb,resources = self.process()
266 nb,resources = self.process()
294 return self.template.render(nb=nb, inlining=inlining), resources
267 return self.template.render(nb=nb, inlining=inlining), resources
295
268
296
269
297 def read(self, filename):
270 def read(self, filename):
298 "read and parse notebook into NotebookNode called self.nb"
271 "read and parse notebook into NotebookNode called self.nb"
299 with io.open(filename) as f:
272 with io.open(filename) as f:
300 self.nb = nbformat.read(f, 'json')
273 self.nb = nbformat.read(f, 'json')
301
274
General Comments 0
You need to be logged in to leave comments. Login now