##// END OF EJS Templates
some improvement
Matthias BUSSONNIER -
Show More
@@ -1,277 +1,289 b''
1 1 """Base classes for the notebook conversion pipeline.
2 2
3 3 This module defines Converter, from which all objects designed to implement
4 4 a conversion of IPython notebooks to some other format should inherit.
5 5 """
6 6 #-----------------------------------------------------------------------------
7 7 # Copyright (c) 2012, the IPython Development Team.
8 8 #
9 9 # Distributed under the terms of the Modified BSD License.
10 10 #
11 11 # The full license is in the file COPYING.txt, distributed with this software.
12 12 #-----------------------------------------------------------------------------
13 13
14 14 #-----------------------------------------------------------------------------
15 15 # Imports
16 16 #-----------------------------------------------------------------------------
17 17
18 18 from __future__ import print_function, absolute_import
19 19
20 20 # Stdlib imports
21 21 import io
22 22 import os
23 23 import re
24 24 from IPython.utils import path
25 25
26 26 from jinja2 import Environment, FileSystemLoader
27 27 env = Environment(
28 28 loader=FileSystemLoader('./templates/'),
29 29 extensions=['jinja2.ext.loopcontrols']
30 30 )
31 31
32 32 texenv = Environment(
33 33 loader=FileSystemLoader('./templates/tex/'),
34 34 extensions=['jinja2.ext.loopcontrols']
35 35 )
36 36
37 37 # IPython imports
38 38 from IPython.nbformat import current as nbformat
39 39 from IPython.config.configurable import Configurable
40 40 from IPython.utils.traitlets import ( Unicode, Any, List)
41 41
42 42 # Our own imports
43 43 from IPython.utils.text import indent
44 44 from .utils import remove_ansi
45 45 from markdown import markdown
46 46 from .utils import highlight, ansi2html
47 47 from .utils import markdown2latex
48 48 #-----------------------------------------------------------------------------
49 49 # Class declarations
50 50 #-----------------------------------------------------------------------------
51 51 def rm_fake(strng):
52 52 return strng.replace('/files/', '')
53 53
54 54 class ConversionException(Exception):
55 55 pass
56 56
57 57
58 58 def python_comment(string):
59 59 return '# '+'\n# '.join(string.split('\n'))
60 60
61 61
62 62
63 63 def header_body():
64 64 """Return the body of the header as a list of strings."""
65 65
66 66 from pygments.formatters import HtmlFormatter
67 67
68 68 header = []
69 69 static = os.path.join(path.get_ipython_package_dir(),
70 70 'frontend', 'html', 'notebook', 'static',
71 71 )
72 72 here = os.path.split(os.path.realpath(__file__))[0]
73 73 css = os.path.join(static, 'css')
74 74 for sheet in [
75 75 # do we need jquery and prettify?
76 76 # os.path.join(static, 'jquery', 'css', 'themes', 'base',
77 77 # 'jquery-ui.min.css'),
78 78 # os.path.join(static, 'prettify', 'prettify.css'),
79 79 os.path.join(css, 'boilerplate.css'),
80 80 os.path.join(css, 'fbm.css'),
81 81 os.path.join(css, 'notebook.css'),
82 82 os.path.join(css, 'renderedhtml.css'),
83 83 # our overrides:
84 84 os.path.join(here, '..', 'css', 'static_html.css'),
85 85 ]:
86 86
87 87 with io.open(sheet, encoding='utf-8') as f:
88 88 s = f.read()
89 89 header.append(s)
90 90
91 91 pygments_css = HtmlFormatter().get_style_defs('.highlight')
92 92 header.append(pygments_css)
93 93 return header
94 94
95
95 # todo, make the key part configurable.
96 96 def _new_figure(data, fmt, count):
97 97 """Create a new figure file in the given format.
98 98
99 99 Returns a path relative to the input file.
100 100 """
101 101 figname = '_fig_%02i.%s' % (count, fmt)
102 102
103 103 # Binary files are base64-encoded, SVG is already XML
104 104 if fmt in ('png', 'jpg', 'pdf'):
105 105 data = data.decode('base64')
106 106
107 107 return figname,data
108 108
109 109
110 110
111 111
112 112 inlining = {}
113 113 inlining['css'] = header_body()
114 114
115 115
116 116
117 117
118 118
119 119
120 120 env.filters['pycomment'] = python_comment
121 121 env.filters['indent'] = indent
122 122 env.filters['rm_fake'] = rm_fake
123 123 env.filters['rm_ansi'] = remove_ansi
124 124 env.filters['markdown'] = markdown
125 125 env.filters['highlight'] = highlight
126 126 env.filters['ansi2html'] = ansi2html
127 127
128 128
129 129
130 130 LATEX_SUBS = (
131 131 (re.compile(r'\\'), r'\\textbackslash'),
132 132 (re.compile(r'([{}_#%&$])'), r'\\\1'),
133 133 (re.compile(r'~'), r'\~{}'),
134 134 (re.compile(r'\^'), r'\^{}'),
135 135 (re.compile(r'"'), r"''"),
136 136 (re.compile(r'\.\.\.+'), r'\\ldots'),
137 137 )
138 138
139 139 def escape_tex(value):
140 140 newval = value
141 141 for pattern, replacement in LATEX_SUBS:
142 142 newval = pattern.sub(replacement, newval)
143 143 return newval
144 144
145 145 texenv.block_start_string = '((*'
146 146 texenv.block_end_string = '*))'
147 147 texenv.variable_start_string = '((('
148 148 texenv.variable_end_string = ')))'
149 149 texenv.comment_start_string = '((='
150 150 texenv.comment_end_string = '=))'
151 151 texenv.filters['escape_tex'] = escape_tex
152 152
153 153 texenv.filters['pycomment'] = python_comment
154 154 texenv.filters['indent'] = indent
155 155 texenv.filters['rm_fake'] = rm_fake
156 156 texenv.filters['rm_ansi'] = remove_ansi
157 157 texenv.filters['markdown'] = markdown
158 158 texenv.filters['highlight'] = highlight
159 159 texenv.filters['ansi2html'] = ansi2html
160 160 texenv.filters['markdown2latex'] = markdown2latex
161 markdown2latex
162
163
164 def haspyout_transformer(nb,_):
165 for worksheet in nb.worksheets:
166 for cell in worksheet.cells:
167 cell.type = cell.cell_type
168 cell.haspyout = False
169 for out in cell.get('outputs', []):
170 if out.output_type == 'pyout':
171 cell.haspyout = True
172 break
173 return nb,_
174
175
176 def outline_figure_transformer(nb,other):
177 count=0
178 for worksheet in nb.worksheets:
179 for cell in worksheet.cells:
180 cell.type = cell.cell_type
181 for i,out in enumerate(cell.get('outputs', [])):
182 print('loop outputs',out.output_type)
183 for type in ['html', 'pdf', 'svg', 'latex', 'png', 'jpg', 'jpeg']:
184 if out.hasattr(type):
185 figname,data = _new_figure(out[type], type,count)
186 cell.outputs[i][type] = figname
187 out[type] = figname
188 print('set',type, 'to' ,figname)
189 other[figname] = data
190 count = count+1
191 return nb,other
192 161
162 def cell_preprocessor(function):
163 """ wrap a function to be executed on all cells of a notebook
164
165 wrapped function parameters :
166 cell : the cell
167 other : external resources
168 index : index of the cell
169 """
170 def wrappedfunc(nb,other):
171 for worksheet in nb.worksheets :
172 for index, cell in enumerate(worksheet.cells):
173 worksheet.cells[index],other= function(cell,other,index)
174 return nb,other
175 return wrappedfunc
176
177
178
179 @cell_preprocessor
180 def haspyout_transformer(cell, other, count):
181 """
182 Add a haspyout flag to cell that have it
183
184 Easier for templating, where you can't know in advance
185 wether to write the out prompt
193 186
194 def print_transformer(nb,other):
195 count=0
196 for worksheet in nb.worksheets:
197 for cell in worksheet.cells:
198 cell.type = cell.cell_type
199 for i,out in enumerate(cell.get('outputs', [])):
200 print(cell.outputs)
187 """
188 cell.type = cell.cell_type
189 cell.haspyout = False
190 for out in cell.get('outputs', []):
191 if out.output_type == 'pyout':
192 cell.haspyout = True
193 break
194 return cell,other
195
196
197 @cell_preprocessor
198 def outline_figure_transformer(cell,other,count):
199 for i,out in enumerate(cell.get('outputs', [])):
200 for type in ['html', 'pdf', 'svg', 'latex', 'png', 'jpg', 'jpeg']:
201 if out.hasattr(type):
202 figname,data = _new_figure(out[type], type,count)
203 cell.outputs[i][type] = figname
204 out['key_'+type] = figname
205 other[figname] = data
206 count = count+1
201 207 return nb,other
202 208
209
203 210 class ConverterTemplate(Configurable):
204 211 """ A Jinja2 base converter templates"""
205 212
206 213 display_data_priority = List(['html', 'pdf', 'svg', 'latex', 'png', 'jpg', 'jpeg' , 'text'],
207 214 config=True,
208 215 help= """
209 216 A list of ast.NodeTransformer subclass instances, which will be applied
210 217 to user input before code is run.
211 218 """
212 219 )
220
221 extract_figures = Bool(False,
222 config=True,
223 help= """
224 wether to remove figure data from ipynb and store them in auxiliary
225 dictionnary
226 """
227 )
213 228 #-------------------------------------------------------------------------
214 229 # Instance-level attributes that are set in the constructor for this
215 230 # class.
216 231 #-------------------------------------------------------------------------
217 232 infile = Any()
218 233
219 234
220 235 infile_dir = Unicode()
221 def display_data_priority_changed(self, name, old, new):
222 print('== changed', name,old,new)
223 236
224 237 def filter_data_type(self,output):
225 238 for fmt in self.display_data_priority:
226 239 if fmt in output:
227 240 return [fmt]
228 241
229 242 def __init__(self, tplfile='fullhtml', preprocessors=[], config=None,tex_environement=False, **kw):
230 243 """
231 244 tplfile : jinja template file to process.
232 245
233 246 config: the Configurable confg object to pass around
234 247
235 248 preprocessors: list of function to run on ipynb json data before conversion
236 249 to extract/inline file,
237 250
238 251 """
239 252 self.env = texenv if tex_environement else env
240 self.ext = '.tplx' if tex_environement else '.tpl'
253 self.ext = '.tplx' if tex_environement else '.tpl'
241 254 self.nb = None
242 255 self.preprocessors = preprocessors
243 256 self.preprocessors.append(haspyout_transformer)
244 257 self.preprocessors.append(outline_figure_transformer)
245 self.preprocessors.append(print_transformer)
246 258 super(ConverterTemplate, self).__init__(config=config, **kw)
247 259 self.env.filters['filter_data_type'] = self.filter_data_type
248 260 self.template = self.env.get_template(tplfile+self.ext)
249 261
250 262
251 263
252 264 def process(self):
253 265 """
254 preprocess the notebook json for easier use with the templates.
255 will call all the `preprocessor`s in order before returning it.
266 preprocess the notebook json for easier use with the templates.
267 will call all the `preprocessor`s in order before returning it.
256 268 """
257 269 nb = self.nb
258 270
259 271 for preprocessor in self.preprocessors:
260 272 nb,others = preprocessor(nb,{})
261 273
262 274 return nb
263 275
264 276 def convert(self):
265 277 """ convert the ipynb file
266 278
267 279 return both the converted ipynb file and a dict containing potential
268 280 other resources
269 281 """
270 282 return self.template.render(nb=self.process(), inlining=inlining), {}
271 283
272 284
273 285 def read(self, filename):
274 286 "read and parse notebook into NotebookNode called self.nb"
275 287 with io.open(filename) as f:
276 288 self.nb = nbformat.read(f, 'json')
277 289
@@ -1,208 +1,207 b''
1 1 ((*- extends 'display_priority.tplx' -*))
2 2
3 3 ((* block in_prompt *))((* endblock in_prompt *))
4 4
5 5 ((* block output_prompt *))((* endblock output_prompt *))
6 6
7 7 ((* block codecell *))\begin{codecell}((( super() )))
8 8 \end{codecell}
9 9 ((* endblock *))
10 10
11 11 ((* block input *))
12 12 \begin{codeinput}
13 13 \begin{lstlisting}
14 14 ((( cell.input )))
15 15 \end{lstlisting}
16 16 \end{codeinput}
17 17 ((* endblock input *))
18 18
19 19
20 20 ((= Those Two are for error displaying
21 21 even if the first one seem to do nothing,
22 22 it introduces a new line
23 23
24 24 =))
25 25 ((* block pyerr *))((( super() )))
26 26 ((* endblock pyerr *))
27 27
28 28 ((* block traceback_line *))
29 29 ((( line |indent| rm_ansi )))((* endblock traceback_line *))
30 30 ((= .... =))
31 31
32 32
33 33 ((*- block output_group -*))
34 34 \begin{codeoutput}
35 35 ((( super() )))
36 36 \end{codeoutput}((* endblock *))
37 37
38 38 ((*- block data_png -*))
39 ++(((output.png)))++
40 39 \begin{center}
41 40 \includegraphics[width=0.7\textwidth]{(((output.png)))}
42 41 \par
43 42 \end{center}
44 43 ((*- endblock -*))
45 44
46 45 ((* block pyout *))
47 46 ((( output.text)))
48 47 ((* endblock pyout *))
49 48
50 49 ((* block data_text *))
51 50 \begin{verbatim}
52 51 ((( output.text )))
53 52 \end{verbatim}
54 53 ((* endblock *))
55 54
56 55 ((* block stream *))
57 56 \begin{verbatim}
58 57 ((( output.text)))
59 58 \end{verbatim}
60 59 ((* endblock stream *))
61 60
62 61
63 62
64 63
65 64 ((* block markdowncell scoped *))((( cell.source | markdown2latex )))
66 65 ((* endblock markdowncell *))
67 66
68 67 ((* block headingcell scoped *))
69 68 \section{((( cell.source)))}
70 69 ((* endblock headingcell *))
71 70
72 71 ((* block rawcell scoped *))
73 72 ((( cell.source | pycomment )))
74 73 ((* endblock rawcell *))
75 74
76 75 ((* block unknowncell scoped *))
77 76 unknown type (((cell.type)))
78 77 ((* endblock unknowncell *))
79 78
80 79
81 80
82 81 ((* block body *))\begin{document}
83 82 ((( super() )))
84 83 \end{document}
85 84 ((* endblock*))
86 85
87 86 ((* block header *))
88 87 %% This file was auto-generated by IPython.
89 88 %% Conversion from the original notebook file:
90 89 %% tests/ipynbref/Gun_Data.orig.ipynb
91 90 %%
92 91 \documentclass[11pt,english]{article}
93 92
94 93 %% This is the automatic preamble used by IPython. Note that it does *not*
95 94 %% include a documentclass declaration, that is added at runtime to the overall
96 95 %% document.
97 96
98 97 \usepackage{amsmath}
99 98 \usepackage{amssymb}
100 99 \usepackage{graphicx}
101 100 \usepackage{ucs}
102 101 \usepackage[utf8x]{inputenc}
103 102
104 103 % needed for markdown enumerations to work
105 104 \usepackage{enumerate}
106 105
107 106 % Slightly bigger margins than the latex defaults
108 107 \usepackage{geometry}
109 108 \geometry{verbose,tmargin=3cm,bmargin=3cm,lmargin=2.5cm,rmargin=2.5cm}
110 109
111 110 % Define a few colors for use in code, links and cell shading
112 111 \usepackage{color}
113 112 \definecolor{orange}{cmyk}{0,0.4,0.8,0.2}
114 113 \definecolor{darkorange}{rgb}{.71,0.21,0.01}
115 114 \definecolor{darkgreen}{rgb}{.12,.54,.11}
116 115 \definecolor{myteal}{rgb}{.26, .44, .56}
117 116 \definecolor{gray}{gray}{0.45}
118 117 \definecolor{lightgray}{gray}{.95}
119 118 \definecolor{mediumgray}{gray}{.8}
120 119 \definecolor{inputbackground}{rgb}{.95, .95, .85}
121 120 \definecolor{outputbackground}{rgb}{.95, .95, .95}
122 121 \definecolor{traceback}{rgb}{1, .95, .95}
123 122
124 123 % Framed environments for code cells (inputs, outputs, errors, ...). The
125 124 % various uses of \unskip (or not) at the end were fine-tuned by hand, so don't
126 125 % randomly change them unless you're sure of the effect it will have.
127 126 \usepackage{framed}
128 127
129 128 % remove extraneous vertical space in boxes
130 129 \setlength\fboxsep{0pt}
131 130
132 131 % codecell is the whole input+output set of blocks that a Code cell can
133 132 % generate.
134 133
135 134 % TODO: unfortunately, it seems that using a framed codecell environment breaks
136 135 % the ability of the frames inside of it to be broken across pages. This
137 136 % causes at least the problem of having lots of empty space at the bottom of
138 137 % pages as new frames are moved to the next page, and if a single frame is too
139 138 % long to fit on a page, will completely stop latex from compiling the
140 139 % document. So unless we figure out a solution to this, we'll have to instead
141 140 % leave the codecell env. as empty. I'm keeping the original codecell
142 141 % definition here (a thin vertical bar) for reference, in case we find a
143 142 % solution to the page break issue.
144 143
145 144 %% \newenvironment{codecell}{%
146 145 %% \def\FrameCommand{\color{mediumgray} \vrule width 1pt \hspace{5pt}}%
147 146 %% \MakeFramed{\vspace{-0.5em}}}
148 147 %% {\unskip\endMakeFramed}
149 148
150 149 % For now, make this a no-op...
151 150 \newenvironment{codecell}{}
152 151
153 152 \newenvironment{codeinput}{%
154 153 \def\FrameCommand{\colorbox{inputbackground}}%
155 154 \MakeFramed{\advance\hsize-\width \FrameRestore}}
156 155 {\unskip\endMakeFramed}
157 156
158 157 \newenvironment{codeoutput}{%
159 158 \def\FrameCommand{\colorbox{outputbackground}}%
160 159 \vspace{-1.4em}
161 160 \MakeFramed{\advance\hsize-\width \FrameRestore}}
162 161 {\unskip\medskip\endMakeFramed}
163 162
164 163 \newenvironment{traceback}{%
165 164 \def\FrameCommand{\colorbox{traceback}}%
166 165 \MakeFramed{\advance\hsize-\width \FrameRestore}}
167 166 {\endMakeFramed}
168 167
169 168 % Use and configure listings package for nicely formatted code
170 169 \usepackage{listingsutf8}
171 170 \lstset{
172 171 language=python,
173 172 inputencoding=utf8x,
174 173 extendedchars=\true,
175 174 aboveskip=\smallskipamount,
176 175 belowskip=\smallskipamount,
177 176 xleftmargin=2mm,
178 177 breaklines=true,
179 178 basicstyle=\small \ttfamily,
180 179 showstringspaces=false,
181 180 keywordstyle=\color{blue}\bfseries,
182 181 commentstyle=\color{myteal},
183 182 stringstyle=\color{darkgreen},
184 183 identifierstyle=\color{darkorange},
185 184 columns=fullflexible, % tighter character kerning, like verb
186 185 }
187 186
188 187 % The hyperref package gives us a pdf with properly built
189 188 % internal navigation ('pdf bookmarks' for the table of contents,
190 189 % internal cross-reference links, web links for URLs, etc.)
191 190 \usepackage{hyperref}
192 191 \hypersetup{
193 192 breaklinks=true, % so long urls are correctly broken across lines
194 193 colorlinks=true,
195 194 urlcolor=blue,
196 195 linkcolor=darkorange,
197 196 citecolor=darkgreen,
198 197 }
199 198
200 199 % hardcode size of all verbatim environments to be a bit smaller
201 200 \makeatletter
202 201 \g@addto@macro\@verbatim\small\topsep=0.5em\partopsep=0pt
203 202 \makeatother
204 203
205 204 % Prevent overflowing lines due to urls and other hard-to-break entities.
206 205 \sloppy
207 206
208 207 ((* endblock *))
General Comments 0
You need to be logged in to leave comments. Login now