Show More
@@ -1,274 +1,257 b'' | |||||
1 | """Exporter for the notebook conversion pipeline. |
|
1 | """Exporter for the notebook conversion pipeline. | |
2 |
|
2 | |||
3 | This module defines Exporter, a highly configurable converter |
|
3 | This module defines Exporter, a highly configurable converter | |
4 | that uses Jinja2 to export notebook files into different format. |
|
4 | that uses Jinja2 to export notebook files into different format. | |
5 |
|
5 | |||
6 | You can register both pre-transformers that will act on the notebook format |
|
6 | You can register both pre-transformers that will act on the notebook format | |
7 | befor conversion and jinja filter that would then be availlable in the templates |
|
7 | befor conversion and jinja filter that would then be availlable in the templates | |
8 | """ |
|
8 | """ | |
9 |
|
9 | |||
10 | #----------------------------------------------------------------------------- |
|
10 | #----------------------------------------------------------------------------- | |
11 | # Copyright (c) 2013, the IPython Development Team. |
|
11 | # Copyright (c) 2013, the IPython Development Team. | |
12 | # |
|
12 | # | |
13 | # Distributed under the terms of the Modified BSD License. |
|
13 | # Distributed under the terms of the Modified BSD License. | |
14 | # |
|
14 | # | |
15 | # The full license is in the file COPYING.txt, distributed with this software. |
|
15 | # The full license is in the file COPYING.txt, distributed with this software. | |
16 | #----------------------------------------------------------------------------- |
|
16 | #----------------------------------------------------------------------------- | |
17 |
|
17 | |||
18 | #----------------------------------------------------------------------------- |
|
18 | #----------------------------------------------------------------------------- | |
19 | # Imports |
|
19 | # Imports | |
20 | #----------------------------------------------------------------------------- |
|
20 | #----------------------------------------------------------------------------- | |
21 | from __future__ import print_function, absolute_import |
|
21 | from __future__ import print_function, absolute_import | |
22 |
|
22 | |||
23 | # Stdlib imports |
|
23 | # Stdlib imports | |
24 | import io |
|
24 | import io | |
25 | import os |
|
25 | import os | |
26 |
|
26 | |||
27 | # IPython imports |
|
27 | # IPython imports | |
28 | from IPython.config.configurable import Configurable |
|
28 | from IPython.config.configurable import Configurable | |
29 | from IPython.nbformat import current as nbformat |
|
29 | from IPython.nbformat import current as nbformat | |
30 | from IPython.utils.traitlets import MetaHasTraits, Unicode, List, Bool |
|
30 | from IPython.utils.traitlets import MetaHasTraits, Unicode, List, Bool | |
31 | from IPython.utils.text import indent |
|
31 | from IPython.utils.text import indent | |
32 |
|
32 | |||
33 | # other libs/dependencies |
|
33 | # other libs/dependencies | |
34 | from jinja2 import Environment, FileSystemLoader |
|
34 | from jinja2 import Environment, FileSystemLoader | |
35 | from markdown import markdown |
|
35 | from markdown import markdown | |
36 |
|
36 | |||
37 | # local import |
|
37 | # local import | |
38 | import filters.strings |
|
38 | import filters.strings | |
39 | import filters.markdown |
|
39 | import filters.markdown | |
40 | import filters.latex |
|
40 | import filters.latex | |
41 | import filters.datatypefilter |
|
41 | import filters.datatypefilter | |
42 | import filters.pygments |
|
42 | import filters.pygments | |
43 | import filters.ansi |
|
43 | import filters.ansi | |
44 |
|
44 | |||
45 | import transformers.extractfigure |
|
45 | import transformers.extractfigure | |
46 | import transformers.csshtmlheader |
|
46 | import transformers.csshtmlheader | |
47 | import transformers.revealhelp |
|
47 | import transformers.revealhelp | |
48 | import transformers.coalescestreams |
|
48 | import transformers.coalescestreams | |
49 |
|
49 | |||
50 |
|
50 | |||
51 | #----------------------------------------------------------------------------- |
|
51 | #----------------------------------------------------------------------------- | |
52 | # Globals and constants |
|
52 | # Globals and constants | |
53 | #----------------------------------------------------------------------------- |
|
53 | #----------------------------------------------------------------------------- | |
54 |
|
54 | |||
55 | #Standard Jinja2 environment constants |
|
55 | #Standard Jinja2 environment constants | |
56 | TEMPLATE_PATH = "/../templates/" |
|
56 | TEMPLATE_PATH = "/../templates/" | |
57 | TEMPLATE_SKELETON_PATH = "/../templates/skeleton/" |
|
57 | TEMPLATE_SKELETON_PATH = "/../templates/skeleton/" | |
58 | TEMPLATE_EXTENSION = ".tpl" |
|
58 | TEMPLATE_EXTENSION = ".tpl" | |
59 |
|
59 | |||
60 | #Jinja2 extensions to load. |
|
60 | #Jinja2 extensions to load. | |
61 | JINJA_EXTENSIONS = ['jinja2.ext.loopcontrols'] |
|
61 | JINJA_EXTENSIONS = ['jinja2.ext.loopcontrols'] | |
62 |
|
62 | |||
63 | #----------------------------------------------------------------------------- |
|
63 | #----------------------------------------------------------------------------- | |
64 | # Classes and functions |
|
64 | # Classes and functions | |
65 | #----------------------------------------------------------------------------- |
|
65 | #----------------------------------------------------------------------------- | |
66 | class Exporter(Configurable): |
|
66 | class Exporter(Configurable): | |
67 | """ A Jinja2 base converter templates |
|
67 | """ A Jinja2 base converter templates | |
68 |
|
68 | |||
69 | Pre-process the IPYNB files, feed it through Jinja2 templates, |
|
69 | Pre-process the IPYNB files, feed it through Jinja2 templates, | |
70 | and spit an converted files and a data object with other data |
|
70 | and spit an converted files and a data object with other data | |
71 | should be mostly configurable |
|
71 | should be mostly configurable | |
72 | """ |
|
72 | """ | |
73 |
|
73 | |||
74 | pre_transformer_order = List(['haspyout_transformer'], |
|
74 | pre_transformer_order = List(['haspyout_transformer'], | |
75 | config=True, |
|
75 | config=True, | |
76 | help= """ |
|
76 | help= """ | |
77 | An ordered list of pre-transformer to apply to the IPYNB |
|
77 | An ordered list of pre-transformer to apply to the IPYNB | |
78 | file before running through templates |
|
78 | file before running through templates | |
79 | """ |
|
79 | """ | |
80 | ) |
|
80 | ) | |
81 |
|
81 | |||
82 | template_file = Unicode( |
|
82 | template_file = Unicode( | |
83 | '', config=True, |
|
83 | '', config=True, | |
84 | help="Name of the template file to use") |
|
84 | help="Name of the template file to use") | |
85 |
|
85 | |||
86 | fileext = Unicode( |
|
86 | fileext = Unicode( | |
87 | 'txt', config=True, |
|
87 | 'txt', config=True, | |
88 | help="Extension of the file that should be written to disk" |
|
88 | help="Extension of the file that should be written to disk" | |
89 | ) |
|
89 | ) | |
90 |
|
90 | |||
91 | stdout = Bool( |
|
91 | stdout = Bool( | |
92 | True, config=True, |
|
92 | True, config=True, | |
93 | help="""Whether to print the converted IPYNB file to stdout |
|
93 | help="""Whether to print the converted IPYNB file to stdout | |
94 | "use full do diff files without actually writing a new file""" |
|
94 | "use full do diff files without actually writing a new file""" | |
95 | ) |
|
95 | ) | |
96 |
|
96 | |||
97 | write = Bool( |
|
97 | write = Bool( | |
98 | False, config=True, |
|
98 | False, config=True, | |
99 | help="""Should the converted notebook file be written to disk |
|
99 | help="""Should the converted notebook file be written to disk | |
100 | along with potential extracted resources.""" |
|
100 | along with potential extracted resources.""" | |
101 | ) |
|
101 | ) | |
102 |
|
102 | |||
103 | #Processors that process the input data prior to the export, set in the |
|
103 | #Processors that process the input data prior to the export, set in the | |
104 | #constructor for this class. |
|
104 | #constructor for this class. | |
105 | preprocessors = [] |
|
105 | preprocessors = [] | |
106 |
|
106 | |||
107 |
def __init__(self, preprocessors=None, jinja_filters=None, config=None, |
|
107 | def __init__(self, preprocessors=None, jinja_filters=None, config=None, **kw): | |
108 | """ Init a new converter. |
|
108 | """ Init a new converter. | |
109 |
|
109 | |||
110 | config: the Configurable config object to pass around. |
|
110 | config: the Configurable config object to pass around. | |
111 |
|
111 | |||
112 |
preprocessors: dict of **avail |
|
112 | preprocessors: dict of **available** key/value function to run on | |
113 | ipynb json data before conversion to extract/inline file. |
|
113 | ipynb json data before conversion to extract/in-line file. | |
114 | See `transformer.py` and `ConfigurableTransformers` |
|
114 | See `transformer.py` and `ConfigurableTransformers` | |
115 |
|
115 | |||
116 | set the order in which the transformers should apply |
|
116 | set the order in which the transformers should apply | |
117 | with the `pre_transformer_order` trait of this class |
|
117 | with the `pre_transformer_order` trait of this class | |
118 |
|
118 | |||
119 | transformers registerd by this key will take precedence on |
|
119 | transformers registered by this key will take precedence on | |
120 | default one. |
|
120 | default one. | |
121 |
|
121 | |||
122 | jinja_filters: dict of supplementary jinja filter that should be made |
|
122 | jinja_filters: dict of supplementary jinja filter that should be made | |
123 |
avail |
|
123 | available in template. If those are of Configurable Class type, | |
124 | they will be instanciated with the config object as argument. |
|
124 | they will be instanciated with the config object as argument. | |
125 |
|
125 | |||
126 |
user defined filter will overwrite the one avail |
|
126 | user defined filter will overwrite the one available by default. | |
127 | """ |
|
127 | """ | |
128 |
|
||||
129 | #Set the default options for the exporter. |
|
|||
130 | default_config = self.config |
|
|||
131 |
|
||||
132 | #Set properties that must be set in the config class in order to |
|
|||
133 | #propagate to other classes. |
|
|||
134 | default_config.GlobalConfigurable.display_data_priority =['svg', 'png', 'latex', 'jpg', 'jpeg','text'] |
|
|||
135 | default_config.ExtractFigureTransformer.display_data_priority=['svg', 'png', 'latex', 'jpg', 'jpeg','text'] |
|
|||
136 |
|
||||
137 | #Set default properties of the exporter. |
|
|||
138 | #For most (or all cases), the template file name matches the format name. |
|
|||
139 | self.display_data_priority= ['svg', 'png', 'latex', 'jpg', 'jpeg','text'] |
|
|||
140 | self.template_file = export_format |
|
|||
141 |
|
||||
142 | if not config == None: |
|
|||
143 | default_config._merge(config) |
|
|||
144 | config = default_config |
|
|||
145 |
|
128 | |||
146 | #Call the base class constructor |
|
129 | #Call the base class constructor | |
147 | super(Exporter, self).__init__(config=config, **kw) |
|
130 | super(Exporter, self).__init__(config=config, **kw) | |
148 |
|
131 | |||
149 | #Standard environment |
|
132 | #Standard environment | |
150 | self.ext = TEMPLATE_EXTENSION |
|
133 | self.ext = TEMPLATE_EXTENSION | |
151 | self._init_environment() |
|
134 | self._init_environment() | |
152 |
|
135 | |||
153 | #TODO: Implement reflection style methods to get user transformers. |
|
136 | #TODO: Implement reflection style methods to get user transformers. | |
154 | #if not preprocessors is None: |
|
137 | #if not preprocessors is None: | |
155 | # for name in self.pre_transformer_order: |
|
138 | # for name in self.pre_transformer_order: | |
156 | # # get the user-defined transformer first |
|
139 | # # get the user-defined transformer first | |
157 | # transformer = preprocessors.get(name, getattr(trans, name, None)) |
|
140 | # transformer = preprocessors.get(name, getattr(trans, name, None)) | |
158 | # if isinstance(transformer, MetaHasTraits): |
|
141 | # if isinstance(transformer, MetaHasTraits): | |
159 | # transformer = transformer(config=config) |
|
142 | # transformer = transformer(config=config) | |
160 | # self.preprocessors.append(transformer) |
|
143 | # self.preprocessors.append(transformer) | |
161 |
|
144 | |||
162 | #For compatibility, TODO: remove later. |
|
145 | #For compatibility, TODO: remove later. | |
163 | self.preprocessors.append(transformers.coalescestreams.coalesce_streams) |
|
146 | self.preprocessors.append(transformers.coalescestreams.coalesce_streams) | |
164 | self.preprocessors.append(transformers.extractfigure.ExtractFigureTransformer(config=config)) |
|
147 | self.preprocessors.append(transformers.extractfigure.ExtractFigureTransformer(config=config)) | |
165 | self.preprocessors.append(transformers.revealhelp.RevealHelpTransformer(config=config)) |
|
148 | self.preprocessors.append(transformers.revealhelp.RevealHelpTransformer(config=config)) | |
166 | self.preprocessors.append(transformers.csshtmlheader.CSSHtmlHeaderTransformer(config=config)) |
|
149 | self.preprocessors.append(transformers.csshtmlheader.CSSHtmlHeaderTransformer(config=config)) | |
167 |
|
150 | |||
168 | #Add filters to the Jinja2 environment |
|
151 | #Add filters to the Jinja2 environment | |
169 | self._register_filters() |
|
152 | self._register_filters() | |
170 |
|
153 | |||
171 | #Load user filters. Overwrite existing filters if need be. |
|
154 | #Load user filters. Overwrite existing filters if need be. | |
172 | if not jinja_filters is None: |
|
155 | if not jinja_filters is None: | |
173 | for key, user_filter in jinja_filters.iteritems(): |
|
156 | for key, user_filter in jinja_filters.iteritems(): | |
174 | if isinstance(user_filter, MetaHasTraits): |
|
157 | if isinstance(user_filter, MetaHasTraits): | |
175 | self.env.filters[key] = user_filter(config=config) |
|
158 | self.env.filters[key] = user_filter(config=config) | |
176 | else: |
|
159 | else: | |
177 | self.env.filters[key] = user_filter |
|
160 | self.env.filters[key] = user_filter | |
178 |
|
161 | |||
179 | #Load the template file. |
|
162 | #Load the template file. | |
180 | self.template = self.env.get_template(self.template_file+self.ext) |
|
163 | self.template = self.env.get_template(self.template_file+self.ext) | |
181 |
|
164 | |||
182 |
|
165 | |||
183 | def from_notebook_node(self, nb): |
|
166 | def from_notebook_node(self, nb): | |
184 | """Export NotebookNode instance |
|
167 | """Export NotebookNode instance | |
185 |
|
168 | |||
186 | nb: NotebookNode to export. |
|
169 | nb: NotebookNode to export. | |
187 |
|
170 | |||
188 | Returns both the converted ipynb file and a dict containing the |
|
171 | Returns both the converted ipynb file and a dict containing the | |
189 | resources created along the way via the transformers and Jinja2 |
|
172 | resources created along the way via the transformers and Jinja2 | |
190 | processing. |
|
173 | processing. | |
191 | """ |
|
174 | """ | |
192 |
|
175 | |||
193 | nb, resources = self._preprocess(nb) |
|
176 | nb, resources = self._preprocess(nb) | |
194 | return self.template.render(nb=nb, resources=resources), resources |
|
177 | return self.template.render(nb=nb, resources=resources), resources | |
195 |
|
178 | |||
196 |
|
179 | |||
197 | def from_filename(self, filename): |
|
180 | def from_filename(self, filename): | |
198 | """Read and export a notebook from a filename |
|
181 | """Read and export a notebook from a filename | |
199 |
|
182 | |||
200 | filename: Filename of the notebook file to export. |
|
183 | filename: Filename of the notebook file to export. | |
201 |
|
184 | |||
202 | Returns both the converted ipynb file and a dict containing the |
|
185 | Returns both the converted ipynb file and a dict containing the | |
203 | resources created along the way via the transformers and Jinja2 |
|
186 | resources created along the way via the transformers and Jinja2 | |
204 | processing. |
|
187 | processing. | |
205 | """ |
|
188 | """ | |
206 | with io.open(filename) as f: |
|
189 | with io.open(filename) as f: | |
207 | return self.from_notebook_node(nbformat.read(f, 'json')) |
|
190 | return self.from_notebook_node(nbformat.read(f, 'json')) | |
208 |
|
191 | |||
209 |
|
192 | |||
210 | def from_file(self, file_stream): |
|
193 | def from_file(self, file_stream): | |
211 | """Read and export a notebook from a file stream |
|
194 | """Read and export a notebook from a file stream | |
212 |
|
195 | |||
213 | file_stream: File handle of file that contains notebook data. |
|
196 | file_stream: File handle of file that contains notebook data. | |
214 |
|
197 | |||
215 | Returns both the converted ipynb file and a dict containing the |
|
198 | Returns both the converted ipynb file and a dict containing the | |
216 | resources created along the way via the transformers and Jinja2 |
|
199 | resources created along the way via the transformers and Jinja2 | |
217 | processing. |
|
200 | processing. | |
218 | """ |
|
201 | """ | |
219 |
|
202 | |||
220 | return self.from_notebook_node(nbformat.read(file_stream, 'json')) |
|
203 | return self.from_notebook_node(nbformat.read(file_stream, 'json')) | |
221 |
|
204 | |||
222 |
|
205 | |||
223 | def register_filter(self, name, filter): |
|
206 | def register_filter(self, name, filter): | |
224 | if MetaHasTraits(filter): |
|
207 | if MetaHasTraits(filter): | |
225 | self.env.filters[name] = filter(config=self.config) |
|
208 | self.env.filters[name] = filter(config=self.config) | |
226 | else: |
|
209 | else: | |
227 | self.env.filters[name] = filter |
|
210 | self.env.filters[name] = filter | |
228 |
|
211 | |||
229 |
|
212 | |||
230 | def _register_filters(self): |
|
213 | def _register_filters(self): | |
231 | self.register_filter('indent', indent) |
|
214 | self.register_filter('indent', indent) | |
232 | self.register_filter('markdown', markdown) |
|
215 | self.register_filter('markdown', markdown) | |
233 | self.register_filter('ansi2html', filters.ansi.ansi2html) |
|
216 | self.register_filter('ansi2html', filters.ansi.ansi2html) | |
234 | self.register_filter('filter_data_type', filters.datatypefilter.DataTypeFilter) |
|
217 | self.register_filter('filter_data_type', filters.datatypefilter.DataTypeFilter) | |
235 | self.register_filter('get_lines', filters.strings.get_lines) |
|
218 | self.register_filter('get_lines', filters.strings.get_lines) | |
236 | self.register_filter('highlight', filters.pygments.highlight) |
|
219 | self.register_filter('highlight', filters.pygments.highlight) | |
237 | self.register_filter('highlight2html', filters.pygments.highlight) |
|
220 | self.register_filter('highlight2html', filters.pygments.highlight) | |
238 | self.register_filter('highlight2latex', filters.pygments.highlight2latex) |
|
221 | self.register_filter('highlight2latex', filters.pygments.highlight2latex) | |
239 | self.register_filter('markdown2latex', filters.markdown.markdown2latex) |
|
222 | self.register_filter('markdown2latex', filters.markdown.markdown2latex) | |
240 | self.register_filter('markdown2rst', filters.markdown.markdown2rst) |
|
223 | self.register_filter('markdown2rst', filters.markdown.markdown2rst) | |
241 | self.register_filter('pycomment', filters.strings.python_comment) |
|
224 | self.register_filter('pycomment', filters.strings.python_comment) | |
242 | self.register_filter('rm_ansi', filters.ansi.remove_ansi) |
|
225 | self.register_filter('rm_ansi', filters.ansi.remove_ansi) | |
243 | self.register_filter('rm_dollars', filters.strings.strip_dollars) |
|
226 | self.register_filter('rm_dollars', filters.strings.strip_dollars) | |
244 | self.register_filter('rm_fake', filters.strings.rm_fake) |
|
227 | self.register_filter('rm_fake', filters.strings.rm_fake) | |
245 | self.register_filter('rm_math_space', filters.latex.rm_math_space) |
|
228 | self.register_filter('rm_math_space', filters.latex.rm_math_space) | |
246 | self.register_filter('wrap', filters.strings.wrap) |
|
229 | self.register_filter('wrap', filters.strings.wrap) | |
247 |
|
230 | |||
248 |
|
231 | |||
249 | def _init_environment(self): |
|
232 | def _init_environment(self): | |
250 | self.env = Environment( |
|
233 | self.env = Environment( | |
251 | loader=FileSystemLoader([ |
|
234 | loader=FileSystemLoader([ | |
252 | os.path.dirname(os.path.realpath(__file__)) + TEMPLATE_PATH, |
|
235 | os.path.dirname(os.path.realpath(__file__)) + TEMPLATE_PATH, | |
253 | os.path.dirname(os.path.realpath(__file__)) + TEMPLATE_SKELETON_PATH, |
|
236 | os.path.dirname(os.path.realpath(__file__)) + TEMPLATE_SKELETON_PATH, | |
254 | ]), |
|
237 | ]), | |
255 | extensions=JINJA_EXTENSIONS |
|
238 | extensions=JINJA_EXTENSIONS | |
256 | ) |
|
239 | ) | |
257 |
|
240 | |||
258 |
|
241 | |||
259 | def _preprocess(self, nb): |
|
242 | def _preprocess(self, nb): | |
260 | """ Preprocess the notebook using the transformers specific |
|
243 | """ Preprocess the notebook using the transformers specific | |
261 | for the current export format. |
|
244 | for the current export format. | |
262 |
|
245 | |||
263 | nb: Notebook to preprocess |
|
246 | nb: Notebook to preprocess | |
264 | """ |
|
247 | """ | |
265 |
|
248 | |||
266 | #Dict of 'resources' that can be filled by the preprocessors. |
|
249 | #Dict of 'resources' that can be filled by the preprocessors. | |
267 | resources = {} |
|
250 | resources = {} | |
268 |
|
251 | |||
269 | #Run each transformer on the notebook. Carry the output along |
|
252 | #Run each transformer on the notebook. Carry the output along | |
270 | #to each transformer |
|
253 | #to each transformer | |
271 | for transformer in self.preprocessors: |
|
254 | for transformer in self.preprocessors: | |
272 | nb, resources = transformer(nb, resources) |
|
255 | nb, resources = transformer(nb, resources) | |
273 | return nb, resources |
|
256 | return nb, resources | |
274 |
|
257 |
General Comments 0
You need to be logged in to leave comments.
Login now