##// END OF EJS Templates
mofe filter in separate file
Matthias BUSSONNIER -
Show More
@@ -0,0 +1,42 b''
1 #-----------------------------------------------------------------------------
2 # Copyright (c) 2012, the IPython Development Team.
3 #
4 # Distributed under the terms of the Modified BSD License.
5 #
6 # The full license is in the file COPYING.txt, distributed with this software.
7 #-----------------------------------------------------------------------------
8
9 from __future__ import absolute_import
10
11 # Stdlib imports
12 import re
13
14 from IPython.utils.text import indent
15 from markdown import markdown
16 from .utils import remove_ansi
17 from .utils import highlight, ansi2html
18 from .utils import markdown2latex
19 #-----------------------------------------------------------------------------
20 # Class declarations
21 #-----------------------------------------------------------------------------
22 def rm_fake(strng):
23 return strng.replace('/files/', '')
24
25 def python_comment(string):
26 return '# '+'\n# '.join(string.split('\n'))
27
28 LATEX_SUBS = (
29 (re.compile(r'\\'), r'\\textbackslash'),
30 (re.compile(r'([{}_#%&$])'), r'\\\1'),
31 (re.compile(r'~'), r'\~{}'),
32 (re.compile(r'\^'), r'\^{}'),
33 (re.compile(r'"'), r"''"),
34 (re.compile(r'\.\.\.+'), r'\\ldots'),
35 )
36
37 def escape_tex(value):
38 newval = value
39 for pattern, replacement in LATEX_SUBS:
40 newval = pattern.sub(replacement, newval)
41 return newval
42
@@ -1,274 +1,219 b''
1 """Base classes for the notebook conversion pipeline.
1 """Base classes for the notebook conversion pipeline.
2
2
3 This module defines Converter, from which all objects designed to implement
3 This module defines Converter, from which all objects designed to implement
4 a conversion of IPython notebooks to some other format should inherit.
4 a conversion of IPython notebooks to some other format should inherit.
5 """
5 """
6 #-----------------------------------------------------------------------------
6 #-----------------------------------------------------------------------------
7 # Copyright (c) 2012, the IPython Development Team.
7 # Copyright (c) 2012, the IPython Development Team.
8 #
8 #
9 # Distributed under the terms of the Modified BSD License.
9 # Distributed under the terms of the Modified BSD License.
10 #
10 #
11 # The full license is in the file COPYING.txt, distributed with this software.
11 # The full license is in the file COPYING.txt, distributed with this software.
12 #-----------------------------------------------------------------------------
12 #-----------------------------------------------------------------------------
13
13
14 #-----------------------------------------------------------------------------
14 #-----------------------------------------------------------------------------
15 # Imports
15 # Imports
16 #-----------------------------------------------------------------------------
16 #-----------------------------------------------------------------------------
17
17
18 from __future__ import print_function, absolute_import
18 from __future__ import print_function, absolute_import
19 from .transformers import extract_figure_transformer
19 from .transformers import extract_figure_transformer
20 import converters.transformers as trans
20 import converters.transformers as trans
21 from converters.jinja_filters import (python_comment, indent,
22 rm_fake, remove_ansi, markdown, highlight,
23 ansi2html, markdown2latex, escape_tex)
21
24
22
25
23 # Stdlib imports
26 # Stdlib imports
24 import io
27 import io
25 import os
28 import os
26 import re
27 from IPython.utils import path
29 from IPython.utils import path
28
30
29 from jinja2 import Environment, FileSystemLoader
31 from jinja2 import Environment, FileSystemLoader
30 env = Environment(
32 env = Environment(
31 loader=FileSystemLoader([
33 loader=FileSystemLoader([
32 './templates/',
34 './templates/',
33 './templates/skeleton/',
35 './templates/skeleton/',
34 ]),
36 ]),
35 extensions=['jinja2.ext.loopcontrols']
37 extensions=['jinja2.ext.loopcontrols']
36 )
38 )
37
39
38 texenv = Environment(
40 texenv = Environment(
39 loader=FileSystemLoader([
41 loader=FileSystemLoader([
40 './templates/tex/',
42 './templates/tex/',
41 './templates/skeleton/tex/',
43 './templates/skeleton/tex/',
42 ]),
44 ]),
43 extensions=['jinja2.ext.loopcontrols']
45 extensions=['jinja2.ext.loopcontrols']
44 )
46 )
45
47
46 # IPython imports
48 # IPython imports
47 from IPython.nbformat import current as nbformat
49 from IPython.nbformat import current as nbformat
48 from IPython.config.configurable import Configurable
50 from IPython.config.configurable import Configurable
49 from IPython.utils.traitlets import ( Unicode, Any, List, Bool)
51 from IPython.utils.traitlets import ( Unicode, Any, List, Bool)
50
52
51 # Our own imports
52 from IPython.utils.text import indent
53 from .utils import remove_ansi
54 from markdown import markdown
55 from .utils import highlight, ansi2html
56 from .utils import markdown2latex
57 #-----------------------------------------------------------------------------
53 #-----------------------------------------------------------------------------
58 # Class declarations
54 # Class declarations
59 #-----------------------------------------------------------------------------
55 #-----------------------------------------------------------------------------
60 def rm_fake(strng):
61 return strng.replace('/files/', '')
62
63 class ConversionException(Exception):
56 class ConversionException(Exception):
64 pass
57 pass
65
58
66
59
67 def python_comment(string):
68 return '# '+'\n# '.join(string.split('\n'))
69
70
71
72 def header_body():
60 def header_body():
73 """Return the body of the header as a list of strings."""
61 """Return the body of the header as a list of strings."""
74
62
75 from pygments.formatters import HtmlFormatter
63 from pygments.formatters import HtmlFormatter
76
64
77 header = []
65 header = []
78 static = os.path.join(path.get_ipython_package_dir(),
66 static = os.path.join(path.get_ipython_package_dir(),
79 'frontend', 'html', 'notebook', 'static',
67 'frontend', 'html', 'notebook', 'static',
80 )
68 )
81 here = os.path.split(os.path.realpath(__file__))[0]
69 here = os.path.split(os.path.realpath(__file__))[0]
82 css = os.path.join(static, 'css')
70 css = os.path.join(static, 'css')
83 for sheet in [
71 for sheet in [
84 # do we need jquery and prettify?
72 # do we need jquery and prettify?
85 # os.path.join(static, 'jquery', 'css', 'themes', 'base',
73 # os.path.join(static, 'jquery', 'css', 'themes', 'base',
86 # 'jquery-ui.min.css'),
74 # 'jquery-ui.min.css'),
87 # os.path.join(static, 'prettify', 'prettify.css'),
75 # os.path.join(static, 'prettify', 'prettify.css'),
88 os.path.join(css, 'boilerplate.css'),
76 os.path.join(css, 'boilerplate.css'),
89 os.path.join(css, 'fbm.css'),
77 os.path.join(css, 'fbm.css'),
90 os.path.join(css, 'notebook.css'),
78 os.path.join(css, 'notebook.css'),
91 os.path.join(css, 'renderedhtml.css'),
79 os.path.join(css, 'renderedhtml.css'),
92 # our overrides:
80 # our overrides:
93 os.path.join(here, '..', 'css', 'static_html.css'),
81 os.path.join(here, '..', 'css', 'static_html.css'),
94 ]:
82 ]:
95
83
96 with io.open(sheet, encoding='utf-8') as f:
84 with io.open(sheet, encoding='utf-8') as f:
97 s = f.read()
85 s = f.read()
98 header.append(s)
86 header.append(s)
99
87
100 pygments_css = HtmlFormatter().get_style_defs('.highlight')
88 pygments_css = HtmlFormatter().get_style_defs('.highlight')
101 header.append(pygments_css)
89 header.append(pygments_css)
102 return header
90 return header
103
91
104 # todo, make the key part configurable.
105 def _new_figure(data, fmt, count):
106 """Create a new figure file in the given format.
107
108 Returns a path relative to the input file.
109 """
110 figname = '_fig_%02i.%s' % (count, fmt)
111
112 # Binary files are base64-encoded, SVG is already XML
113 if fmt in ('png', 'jpg', 'pdf'):
114 data = data.decode('base64')
115
116 return figname,data
117
92
118
93
119
94
120
95
121 inlining = {}
96 inlining = {}
122 inlining['css'] = header_body()
97 inlining['css'] = header_body()
123
98
124 LATEX_SUBS = (
125 (re.compile(r'\\'), r'\\textbackslash'),
126 (re.compile(r'([{}_#%&$])'), r'\\\1'),
127 (re.compile(r'~'), r'\~{}'),
128 (re.compile(r'\^'), r'\^{}'),
129 (re.compile(r'"'), r"''"),
130 (re.compile(r'\.\.\.+'), r'\\ldots'),
131 )
132
99
133 def escape_tex(value):
134 newval = value
135 for pattern, replacement in LATEX_SUBS:
136 newval = pattern.sub(replacement, newval)
137 return newval
138
100
139 texenv.block_start_string = '((*'
101 texenv.block_start_string = '((*'
140 texenv.block_end_string = '*))'
102 texenv.block_end_string = '*))'
141 texenv.variable_start_string = '((('
103 texenv.variable_start_string = '((('
142 texenv.variable_end_string = ')))'
104 texenv.variable_end_string = ')))'
143 texenv.comment_start_string = '((='
105 texenv.comment_start_string = '((='
144 texenv.comment_end_string = '=))'
106 texenv.comment_end_string = '=))'
145 texenv.filters['escape_tex'] = escape_tex
107 texenv.filters['escape_tex'] = escape_tex
146
108
147 def cell_preprocessor(function):
148 """ wrap a function to be executed on all cells of a notebook
149
150 wrapped function parameters :
151 cell : the cell
152 other : external resources
153 index : index of the cell
154 """
155 def wrappedfunc(nb,other):
156 for worksheet in nb.worksheets :
157 for index, cell in enumerate(worksheet.cells):
158 worksheet.cells[index],other= function(cell,other,index)
159 return nb,other
160 return wrappedfunc
161
162
163
164
109
165 class ConverterTemplate(Configurable):
110 class ConverterTemplate(Configurable):
166 """ A Jinja2 base converter templates"""
111 """ A Jinja2 base converter templates"""
167
112
168 display_data_priority = List(['html', 'pdf', 'svg', 'latex', 'png', 'jpg', 'jpeg' , 'text'],
113 display_data_priority = List(['html', 'pdf', 'svg', 'latex', 'png', 'jpg', 'jpeg' , 'text'],
169 config=True,
114 config=True,
170 help= """
115 help= """
171 An ordered list of prefered output type, the firs encounterd will usually be
116 An ordered list of prefered output type, the firs encounterd will usually be
172 used when converting discarding the others.
117 used when converting discarding the others.
173 """
118 """
174 )
119 )
175
120
176 pre_transformer_order = List(['haspyout_transformer'],
121 pre_transformer_order = List(['haspyout_transformer'],
177 config=True,
122 config=True,
178 help= """ An ordered list of pretransformer to apply to the ipynb file befor running through templates
123 help= """ An ordered list of pretransformer to apply to the ipynb file befor running through templates
179 """
124 """
180 )
125 )
181
126
182 extract_figures = Bool(False,
127 extract_figures = Bool(False,
183 config=True,
128 config=True,
184 help= """
129 help= """
185 wether to remove figure data from ipynb and store them in auxiliary
130 wether to remove figure data from ipynb and store them in auxiliary
186 dictionnary
131 dictionnary
187 """
132 """
188 )
133 )
189
134
190 tex_environement = Bool(False,
135 tex_environement = Bool(False,
191 config=True,
136 config=True,
192 help=""" is this a tex environment or not """)
137 help=""" is this a tex environment or not """)
193
138
194 template_file = Unicode('',
139 template_file = Unicode('',
195 config=True,
140 config=True,
196 help=""" whetever """ )
141 help=""" whetever """ )
197 #-------------------------------------------------------------------------
142 #-------------------------------------------------------------------------
198 # Instance-level attributes that are set in the constructor for this
143 # Instance-level attributes that are set in the constructor for this
199 # class.
144 # class.
200 #-------------------------------------------------------------------------
145 #-------------------------------------------------------------------------
201 infile = Any()
146 infile = Any()
202
147
203
148
204 infile_dir = Unicode()
149 infile_dir = Unicode()
205
150
206 def filter_data_type(self,output):
151 def filter_data_type(self, output):
207 for fmt in self.display_data_priority:
152 for fmt in self.display_data_priority:
208 if fmt in output:
153 if fmt in output:
209 return [fmt]
154 return [fmt]
210
155
211 def __init__(self, preprocessors=[], config=None, **kw):
156 def __init__(self, preprocessors=[], config=None, **kw):
212 """
157 """
213 tplfile : jinja template file to process.
158 tplfile : jinja template file to process.
214
159
215 config: the Configurable confg object to pass around
160 config: the Configurable confg object to pass around
216
161
217 preprocessors: list of function to run on ipynb json data before conversion
162 preprocessors: list of function to run on ipynb json data before conversion
218 to extract/inline file,
163 to extract/inline file,
219
164
220 """
165 """
221 super(ConverterTemplate, self).__init__(config=config, **kw)
166 super(ConverterTemplate, self).__init__(config=config, **kw)
222 self.env = texenv if self.tex_environement else env
167 self.env = texenv if self.tex_environement else env
223 self.ext = '.tplx' if self.tex_environement else '.tpl'
168 self.ext = '.tplx' if self.tex_environement else '.tpl'
224 self.nb = None
169 self.nb = None
225 self.preprocessors = preprocessors
170 self.preprocessors = preprocessors
226 for name in self.pre_transformer_order:
171 for name in self.pre_transformer_order:
227 self.preprocessors.append(getattr(trans,'haspyout_transformer'))
172 self.preprocessors.append(getattr(trans, name))
228 if self.extract_figures:
173 if self.extract_figures:
229 self.preprocessors.append(extract_figure_transformer)
174 self.preprocessors.append(extract_figure_transformer)
230
175
231 self.env.filters['filter_data_type'] = self.filter_data_type
176 self.env.filters['filter_data_type'] = self.filter_data_type
232 self.env.filters['pycomment'] = python_comment
177 self.env.filters['pycomment'] = python_comment
233 self.env.filters['indent'] = indent
178 self.env.filters['indent'] = indent
234 self.env.filters['rm_fake'] = rm_fake
179 self.env.filters['rm_fake'] = rm_fake
235 self.env.filters['rm_ansi'] = remove_ansi
180 self.env.filters['rm_ansi'] = remove_ansi
236 self.env.filters['markdown'] = markdown
181 self.env.filters['markdown'] = markdown
237 self.env.filters['highlight'] = highlight
182 self.env.filters['highlight'] = highlight
238 self.env.filters['ansi2html'] = ansi2html
183 self.env.filters['ansi2html'] = ansi2html
239 self.env.filters['markdown2latex'] = markdown2latex
184 self.env.filters['markdown2latex'] = markdown2latex
240
185
241 self.template = self.env.get_template(self.template_file+self.ext)
186 self.template = self.env.get_template(self.template_file+self.ext)
242
187
243
188
244 def process(self):
189 def process(self):
245 """
190 """
246 preprocess the notebook json for easier use with the templates.
191 preprocess the notebook json for easier use with the templates.
247 will call all the `preprocessor`s in order before returning it.
192 will call all the `preprocessor`s in order before returning it.
248 """
193 """
249 nb = self.nb
194 nb = self.nb
250
195
251 # dict of 'resources' that could be made by the preprocessors
196 # dict of 'resources' that could be made by the preprocessors
252 # like key/value data to extract files from ipynb like in latex conversion
197 # like key/value data to extract files from ipynb like in latex conversion
253 resources = {}
198 resources = {}
254
199
255 for preprocessor in self.preprocessors:
200 for preprocessor in self.preprocessors:
256 nb,resources = preprocessor(nb,resources)
201 nb, resources = preprocessor(nb, resources)
257
202
258 return nb, resources
203 return nb, resources
259
204
260 def convert(self):
205 def convert(self):
261 """ convert the ipynb file
206 """ convert the ipynb file
262
207
263 return both the converted ipynb file and a dict containing potential
208 return both the converted ipynb file and a dict containing potential
264 other resources
209 other resources
265 """
210 """
266 nb,resources = self.process()
211 nb, resources = self.process()
267 return self.template.render(nb=nb, inlining=inlining), resources
212 return self.template.render(nb=nb, inlining=inlining), resources
268
213
269
214
270 def read(self, filename):
215 def read(self, filename):
271 "read and parse notebook into NotebookNode called self.nb"
216 "read and parse notebook into NotebookNode called self.nb"
272 with io.open(filename) as f:
217 with io.open(filename) as f:
273 self.nb = nbformat.read(f, 'json')
218 self.nb = nbformat.read(f, 'json')
274
219
@@ -1,50 +1,64 b''
1 """
1 """
2
2
3 """
3 """
4
4
5 def cell_preprocessor(function):
5 def cell_preprocessor(function):
6 """ wrap a function to be executed on all cells of a notebook
6 """ wrap a function to be executed on all cells of a notebook
7
7
8 wrapped function parameters :
8 wrapped function parameters :
9 cell : the cell
9 cell : the cell
10 other : external resources
10 other : external resources
11 index : index of the cell
11 index : index of the cell
12 """
12 """
13 def wrappedfunc(nb,other):
13 def wrappedfunc(nb, other):
14 for worksheet in nb.worksheets :
14 for worksheet in nb.worksheets :
15 for index, cell in enumerate(worksheet.cells):
15 for index, cell in enumerate(worksheet.cells):
16 worksheet.cells[index],other= function(cell,other,index)
16 worksheet.cells[index], other = function(cell, other, index)
17 return nb,other
17 return nb, other
18 return wrappedfunc
18 return wrappedfunc
19
19
20
20
21 @cell_preprocessor
21 @cell_preprocessor
22 def haspyout_transformer(cell, other, count):
22 def haspyout_transformer(cell, other, count):
23 """
23 """
24 Add a haspyout flag to cell that have it
24 Add a haspyout flag to cell that have it
25
25
26 Easier for templating, where you can't know in advance
26 Easier for templating, where you can't know in advance
27 wether to write the out prompt
27 wether to write the out prompt
28
28
29 """
29 """
30 cell.type = cell.cell_type
30 cell.type = cell.cell_type
31 cell.haspyout = False
31 cell.haspyout = False
32 for out in cell.get('outputs', []):
32 for out in cell.get('outputs', []):
33 if out.output_type == 'pyout':
33 if out.output_type == 'pyout':
34 cell.haspyout = True
34 cell.haspyout = True
35 break
35 break
36 return cell,other
36 return cell, other
37
37
38
38
39 # todo, make the key part configurable.
40 def _new_figure(data, fmt, count):
41 """Create a new figure file in the given format.
42
43 Returns a path relative to the input file.
44 """
45 figname = '_fig_%02i.%s' % (count, fmt)
46
47 # Binary files are base64-encoded, SVG is already XML
48 if fmt in ('png', 'jpg', 'pdf'):
49 data = data.decode('base64')
50
51 return figname, data
52
39 @cell_preprocessor
53 @cell_preprocessor
40 def extract_figure_transformer(cell,other,count):
54 def extract_figure_transformer(cell, other, count):
41 for i,out in enumerate(cell.get('outputs', [])):
55 for i, out in enumerate(cell.get('outputs', [])):
42 for type in ['html', 'pdf', 'svg', 'latex', 'png', 'jpg', 'jpeg']:
56 for type in ['html', 'pdf', 'svg', 'latex', 'png', 'jpg', 'jpeg']:
43 if out.hasattr(type):
57 if out.hasattr(type):
44 figname,data = _new_figure(out[type], type, count)
58 figname, data = _new_figure(out[type], type, count)
45 cell.outputs[i][type] = figname
59 cell.outputs[i][type] = figname
46 out['key_'+type] = figname
60 out['key_'+type] = figname
47 other[figname] = data
61 other[figname] = data
48 count = count+1
62 count = count+1
49 return cell,other
63 return cell, other
50
64
General Comments 0
You need to be logged in to leave comments. Login now