##// END OF EJS Templates
Add a more informative error message for IPython directive....
chebee7i -
Show More
@@ -1,1162 +1,1191 b''
1 1 # -*- coding: utf-8 -*-
2 2 """
3 3 Sphinx directive to support embedded IPython code.
4 4
5 5 This directive allows pasting of entire interactive IPython sessions, prompts
6 6 and all, and their code will actually get re-executed at doc build time, with
7 7 all prompts renumbered sequentially. It also allows you to input code as a pure
8 8 python input by giving the argument python to the directive. The output looks
9 9 like an interactive ipython section.
10 10
11 11 To enable this directive, simply list it in your Sphinx ``conf.py`` file
12 12 (making sure the directory where you placed it is visible to sphinx, as is
13 13 needed for all Sphinx directives). For example, to enable syntax highlighting
14 14 and the IPython directive::
15 15
16 16 extensions = ['IPython.sphinxext.ipython_console_highlighting',
17 17 'IPython.sphinxext.ipython_directive']
18 18
19 19 The IPython directive outputs code-blocks with the language 'ipython'. So
20 20 if you do not have the syntax highlighting extension enabled as well, then
21 21 all rendered code-blocks will be uncolored. By default this directive assumes
22 22 that your prompts are unchanged IPython ones, but this can be customized.
23 23 The configurable options that can be placed in conf.py are:
24 24
25 25 ipython_savefig_dir:
26 26 The directory in which to save the figures. This is relative to the
27 27 Sphinx source directory. The default is `html_static_path`.
28 28 ipython_rgxin:
29 29 The compiled regular expression to denote the start of IPython input
30 30 lines. The default is re.compile('In \[(\d+)\]:\s?(.*)\s*'). You
31 31 shouldn't need to change this.
32 32 ipython_rgxout:
33 33 The compiled regular expression to denote the start of IPython output
34 34 lines. The default is re.compile('Out\[(\d+)\]:\s?(.*)\s*'). You
35 35 shouldn't need to change this.
36 36 ipython_promptin:
37 37 The string to represent the IPython input prompt in the generated ReST.
38 38 The default is 'In [%d]:'. This expects that the line numbers are used
39 39 in the prompt.
40 40 ipython_promptout:
41 41 The string to represent the IPython prompt in the generated ReST. The
42 42 default is 'Out [%d]:'. This expects that the line numbers are used
43 43 in the prompt.
44 44 ipython_mplbackend:
45 45 The string which specifies if the embedded Sphinx shell should import
46 46 Matplotlib and set the backend. The value specifies a backend that is
47 47 passed to `matplotlib.use()` before any lines in `ipython_execlines` are
48 48 executed. If not specified in conf.py, then the default value of 'agg' is
49 49 used. To use the IPython directive without matplotlib as a dependency, set
50 50 the value to `None`. It may end up that matplotlib is still imported
51 51 if the user specifies so in `ipython_execlines` or makes use of the
52 52 @savefig pseudo decorator.
53 53 ipython_execlines:
54 54 A list of strings to be exec'd in the embedded Sphinx shell. Typical
55 55 usage is to make certain packages always available. Set this to an empty
56 56 list if you wish to have no imports always available. If specified in
57 57 conf.py as `None`, then it has the effect of making no imports available.
58 58 If omitted from conf.py altogether, then the default value of
59 59 ['import numpy as np', 'import matplotlib.pyplot as plt'] is used.
60 60 ipython_holdcount
61 61 When the @suppress pseudo-decorator is used, the execution count can be
62 62 incremented or not. The default behavior is to hold the execution count,
63 63 corresponding to a value of `True`. Set this to `False` to increment
64 64 the execution count after each suppressed command.
65 65
66 66 As an example, to use the IPython directive when `matplotlib` is not available,
67 67 one sets the backend to `None`::
68 68
69 69 ipython_mplbackend = None
70 70
71 71 An example usage of the directive is:
72 72
73 73 .. code-block:: rst
74 74
75 75 .. ipython::
76 76
77 77 In [1]: x = 1
78 78
79 79 In [2]: y = x**2
80 80
81 81 In [3]: print(y)
82 82
83 83 See http://matplotlib.org/sampledoc/ipython_directive.html for additional
84 84 documentation.
85 85
86 86 Pseudo-Decorators
87 87 =================
88 88
89 89 Note: Only one decorator is supported per input. If more than one decorator
90 90 is specified, then only the last one is used.
91 91
92 92 In addition to the Pseudo-Decorators/options described at the above link,
93 93 several enhancements have been made. The directive will emit a message to the
94 94 console at build-time if code-execution resulted in an exception or warning.
95 95 You can suppress these on a per-block basis by specifying the :okexcept:
96 96 or :okwarning: options:
97 97
98 98 .. code-block:: rst
99 99
100 100 .. ipython::
101 101 :okexcept:
102 102 :okwarning:
103 103
104 104 In [1]: 1/0
105 105 In [2]: # raise warning.
106 106
107 107 ToDo
108 108 ----
109 109
110 110 - Turn the ad-hoc test() function into a real test suite.
111 111 - Break up ipython-specific functionality from matplotlib stuff into better
112 112 separated code.
113 113
114 114 Authors
115 115 -------
116 116
117 117 - John D Hunter: orignal author.
118 118 - Fernando Perez: refactoring, documentation, cleanups, port to 0.11.
119 119 - VΓ‘clavΕ milauer <eudoxos-AT-arcig.cz>: Prompt generalizations.
120 120 - Skipper Seabold, refactoring, cleanups, pure python addition
121 121 """
122 122 from __future__ import print_function
123 123
124 124 #-----------------------------------------------------------------------------
125 125 # Imports
126 126 #-----------------------------------------------------------------------------
127 127
128 128 # Stdlib
129 129 import atexit
130 130 import os
131 131 import re
132 132 import sys
133 133 import tempfile
134 134 import ast
135 135 import warnings
136 136 import shutil
137 137
138 138 # To keep compatibility with various python versions
139 139 try:
140 140 from hashlib import md5
141 141 except ImportError:
142 142 from md5 import md5
143 143
144 144 # Third-party
145 145 import sphinx
146 146 from docutils.parsers.rst import directives
147 147 from docutils import nodes
148 148 from sphinx.util.compat import Directive
149 149
150 150 # Our own
151 151 from traitlets.config import Config
152 152 from IPython import InteractiveShell
153 153 from IPython.core.profiledir import ProfileDir
154 154 from IPython.utils import io
155 155 from IPython.utils.py3compat import PY3
156 156
157 157 if PY3:
158 158 from io import StringIO
159 159 else:
160 160 from StringIO import StringIO
161 161
162 162 #-----------------------------------------------------------------------------
163 163 # Globals
164 164 #-----------------------------------------------------------------------------
165 165 # for tokenizing blocks
166 166 COMMENT, INPUT, OUTPUT = range(3)
167 167
168 168 #-----------------------------------------------------------------------------
169 169 # Functions and class declarations
170 170 #-----------------------------------------------------------------------------
171 171
172 172 def block_parser(part, rgxin, rgxout, fmtin, fmtout):
173 173 """
174 174 part is a string of ipython text, comprised of at most one
175 input, one ouput, comments, and blank lines. The block parser
175 input, one output, comments, and blank lines. The block parser
176 176 parses the text into a list of::
177 177
178 178 blocks = [ (TOKEN0, data0), (TOKEN1, data1), ...]
179 179
180 180 where TOKEN is one of [COMMENT | INPUT | OUTPUT ] and
181 181 data is, depending on the type of token::
182 182
183 183 COMMENT : the comment string
184 184
185 185 INPUT: the (DECORATOR, INPUT_LINE, REST) where
186 186 DECORATOR: the input decorator (or None)
187 187 INPUT_LINE: the input as string (possibly multi-line)
188 188 REST : any stdout generated by the input line (not OUTPUT)
189 189
190 190 OUTPUT: the output string, possibly multi-line
191 191
192 192 """
193 193 block = []
194 194 lines = part.split('\n')
195 195 N = len(lines)
196 196 i = 0
197 197 decorator = None
198 198 while 1:
199 199
200 200 if i==N:
201 201 # nothing left to parse -- the last line
202 202 break
203 203
204 204 line = lines[i]
205 205 i += 1
206 206 line_stripped = line.strip()
207 207 if line_stripped.startswith('#'):
208 208 block.append((COMMENT, line))
209 209 continue
210 210
211 211 if line_stripped.startswith('@'):
212 212 # Here is where we assume there is, at most, one decorator.
213 213 # Might need to rethink this.
214 214 decorator = line_stripped
215 215 continue
216 216
217 217 # does this look like an input line?
218 218 matchin = rgxin.match(line)
219 219 if matchin:
220 220 lineno, inputline = int(matchin.group(1)), matchin.group(2)
221 221
222 222 # the ....: continuation string
223 223 continuation = ' %s:'%''.join(['.']*(len(str(lineno))+2))
224 224 Nc = len(continuation)
225 225 # input lines can continue on for more than one line, if
226 226 # we have a '\' line continuation char or a function call
227 227 # echo line 'print'. The input line can only be
228 228 # terminated by the end of the block or an output line, so
229 229 # we parse out the rest of the input line if it is
230 230 # multiline as well as any echo text
231 231
232 232 rest = []
233 233 while i<N:
234 234
235 235 # look ahead; if the next line is blank, or a comment, or
236 236 # an output line, we're done
237 237
238 238 nextline = lines[i]
239 239 matchout = rgxout.match(nextline)
240 240 #print "nextline=%s, continuation=%s, starts=%s"%(nextline, continuation, nextline.startswith(continuation))
241 241 if matchout or nextline.startswith('#'):
242 242 break
243 243 elif nextline.startswith(continuation):
244 244 # The default ipython_rgx* treat the space following the colon as optional.
245 245 # However, If the space is there we must consume it or code
246 246 # employing the cython_magic extension will fail to execute.
247 247 #
248 248 # This works with the default ipython_rgx* patterns,
249 249 # If you modify them, YMMV.
250 250 nextline = nextline[Nc:]
251 251 if nextline and nextline[0] == ' ':
252 252 nextline = nextline[1:]
253 253
254 254 inputline += '\n' + nextline
255 255 else:
256 256 rest.append(nextline)
257 257 i+= 1
258 258
259 259 block.append((INPUT, (decorator, inputline, '\n'.join(rest))))
260 260 continue
261 261
262 262 # if it looks like an output line grab all the text to the end
263 263 # of the block
264 264 matchout = rgxout.match(line)
265 265 if matchout:
266 266 lineno, output = int(matchout.group(1)), matchout.group(2)
267 267 if i<N-1:
268 268 output = '\n'.join([output] + lines[i:])
269 269
270 270 block.append((OUTPUT, output))
271 271 break
272 272
273 273 return block
274 274
275 275
276 276 class EmbeddedSphinxShell(object):
277 277 """An embedded IPython instance to run inside Sphinx"""
278 278
279 279 def __init__(self, exec_lines=None):
280 280
281 281 self.cout = StringIO()
282 282
283 283 if exec_lines is None:
284 284 exec_lines = []
285 285
286 286 # Create config object for IPython
287 287 config = Config()
288 288 config.InteractiveShell.autocall = False
289 289 config.InteractiveShell.autoindent = False
290 290 config.InteractiveShell.colors = 'NoColor'
291 291
292 292 # create a profile so instance history isn't saved
293 293 tmp_profile_dir = tempfile.mkdtemp(prefix='profile_')
294 294 profname = 'auto_profile_sphinx_build'
295 295 pdir = os.path.join(tmp_profile_dir,profname)
296 296 profile = ProfileDir.create_profile_dir(pdir)
297 297
298 298 # Create and initialize global ipython, but don't start its mainloop.
299 299 # This will persist across different EmbededSphinxShell instances.
300 atexit.register(self.cleanup)
301 300 IP = InteractiveShell.instance(config=config, profile_dir=profile)
301 atexit.register(self.cleanup)
302 302
303 303 # io.stdout redirect must be done after instantiating InteractiveShell
304 304 io.stdout = self.cout
305 305 io.stderr = self.cout
306 306
307 307 # For debugging, so we can see normal output, use this:
308 308 #from IPython.utils.io import Tee
309 309 #io.stdout = Tee(self.cout, channel='stdout') # dbg
310 310 #io.stderr = Tee(self.cout, channel='stderr') # dbg
311 311
312 312 # Store a few parts of IPython we'll need.
313 313 self.IP = IP
314 314 self.user_ns = self.IP.user_ns
315 315 self.user_global_ns = self.IP.user_global_ns
316 316
317 317 self.input = ''
318 318 self.output = ''
319 319 self.tmp_profile_dir = tmp_profile_dir
320 320
321 321 self.is_verbatim = False
322 322 self.is_doctest = False
323 323 self.is_suppress = False
324 324
325 325 # Optionally, provide more detailed information to shell.
326 326 # this is assigned by the SetUp method of IPythonDirective
327 327 # to point at itself.
328 328 #
329 329 # So, you can access handy things at self.directive.state
330 330 self.directive = None
331 331
332 332 # on the first call to the savefig decorator, we'll import
333 333 # pyplot as plt so we can make a call to the plt.gcf().savefig
334 334 self._pyplot_imported = False
335 335
336 336 # Prepopulate the namespace.
337 337 for line in exec_lines:
338 338 self.process_input_line(line, store_history=False)
339 339
340 340 def cleanup(self):
341 341 shutil.rmtree(self.tmp_profile_dir, ignore_errors=True)
342 342
343 343 def clear_cout(self):
344 344 self.cout.seek(0)
345 345 self.cout.truncate(0)
346 346
347 347 def process_input_line(self, line, store_history=True):
348 348 """process the input, capturing stdout"""
349 349
350 350 stdout = sys.stdout
351 351 splitter = self.IP.input_splitter
352 352 try:
353 353 sys.stdout = self.cout
354 354 splitter.push(line)
355 355 more = splitter.push_accepts_more()
356 356 if not more:
357 357 source_raw = splitter.raw_reset()
358 358 self.IP.run_cell(source_raw, store_history=store_history)
359 359 finally:
360 360 sys.stdout = stdout
361 361
362 362 def process_image(self, decorator):
363 363 """
364 364 # build out an image directive like
365 365 # .. image:: somefile.png
366 366 # :width 4in
367 367 #
368 368 # from an input like
369 369 # savefig somefile.png width=4in
370 370 """
371 371 savefig_dir = self.savefig_dir
372 372 source_dir = self.source_dir
373 373 saveargs = decorator.split(' ')
374 374 filename = saveargs[1]
375 375 # insert relative path to image file in source
376 376 outfile = os.path.relpath(os.path.join(savefig_dir,filename),
377 377 source_dir)
378 378
379 379 imagerows = ['.. image:: %s'%outfile]
380 380
381 381 for kwarg in saveargs[2:]:
382 382 arg, val = kwarg.split('=')
383 383 arg = arg.strip()
384 384 val = val.strip()
385 385 imagerows.append(' :%s: %s'%(arg, val))
386 386
387 387 image_file = os.path.basename(outfile) # only return file name
388 388 image_directive = '\n'.join(imagerows)
389 389 return image_file, image_directive
390 390
391 391 # Callbacks for each type of token
392 392 def process_input(self, data, input_prompt, lineno):
393 393 """
394 394 Process data block for INPUT token.
395 395
396 396 """
397 397 decorator, input, rest = data
398 398 image_file = None
399 399 image_directive = None
400 400
401 401 is_verbatim = decorator=='@verbatim' or self.is_verbatim
402 402 is_doctest = (decorator is not None and \
403 403 decorator.startswith('@doctest')) or self.is_doctest
404 404 is_suppress = decorator=='@suppress' or self.is_suppress
405 405 is_okexcept = decorator=='@okexcept' or self.is_okexcept
406 406 is_okwarning = decorator=='@okwarning' or self.is_okwarning
407 407 is_savefig = decorator is not None and \
408 408 decorator.startswith('@savefig')
409 409
410 410 input_lines = input.split('\n')
411 411 if len(input_lines) > 1:
412 412 if input_lines[-1] != "":
413 413 input_lines.append('') # make sure there's a blank line
414 414 # so splitter buffer gets reset
415 415
416 416 continuation = ' %s:'%''.join(['.']*(len(str(lineno))+2))
417 417
418 418 if is_savefig:
419 419 image_file, image_directive = self.process_image(decorator)
420 420
421 421 ret = []
422 422 is_semicolon = False
423 423
424 424 # Hold the execution count, if requested to do so.
425 425 if is_suppress and self.hold_count:
426 426 store_history = False
427 427 else:
428 428 store_history = True
429 429
430 430 # Note: catch_warnings is not thread safe
431 431 with warnings.catch_warnings(record=True) as ws:
432 432 for i, line in enumerate(input_lines):
433 433 if line.endswith(';'):
434 434 is_semicolon = True
435 435
436 436 if i == 0:
437 437 # process the first input line
438 438 if is_verbatim:
439 439 self.process_input_line('')
440 440 self.IP.execution_count += 1 # increment it anyway
441 441 else:
442 442 # only submit the line in non-verbatim mode
443 443 self.process_input_line(line, store_history=store_history)
444 444 formatted_line = '%s %s'%(input_prompt, line)
445 445 else:
446 446 # process a continuation line
447 447 if not is_verbatim:
448 448 self.process_input_line(line, store_history=store_history)
449 449
450 450 formatted_line = '%s %s'%(continuation, line)
451 451
452 452 if not is_suppress:
453 453 ret.append(formatted_line)
454 454
455 455 if not is_suppress and len(rest.strip()) and is_verbatim:
456 456 # The "rest" is the standard output of the input. This needs to be
457 457 # added when in verbatim mode. If there is no "rest", then we don't
458 458 # add it, as the new line will be added by the processed output.
459 459 ret.append(rest)
460 460
461 461 # Fetch the processed output. (This is not the submitted output.)
462 462 self.cout.seek(0)
463 463 processed_output = self.cout.read()
464 464 if not is_suppress and not is_semicolon:
465 465 #
466 466 # In IPythonDirective.run, the elements of `ret` are eventually
467 467 # combined such that '' entries correspond to newlines. So if
468 468 # `processed_output` is equal to '', then the adding it to `ret`
469 469 # ensures that there is a blank line between consecutive inputs
470 470 # that have no outputs, as in:
471 471 #
472 472 # In [1]: x = 4
473 473 #
474 474 # In [2]: x = 5
475 475 #
476 476 # When there is processed output, it has a '\n' at the tail end. So
477 477 # adding the output to `ret` will provide the necessary spacing
478 478 # between consecutive input/output blocks, as in:
479 479 #
480 480 # In [1]: x
481 481 # Out[1]: 5
482 482 #
483 483 # In [2]: x
484 484 # Out[2]: 5
485 485 #
486 486 # When there is stdout from the input, it also has a '\n' at the
487 487 # tail end, and so this ensures proper spacing as well. E.g.:
488 488 #
489 489 # In [1]: print x
490 490 # 5
491 491 #
492 492 # In [2]: x = 5
493 493 #
494 494 # When in verbatim mode, `processed_output` is empty (because
495 495 # nothing was passed to IP. Sometimes the submitted code block has
496 496 # an Out[] portion and sometimes it does not. When it does not, we
497 497 # need to ensure proper spacing, so we have to add '' to `ret`.
498 498 # However, if there is an Out[] in the submitted code, then we do
499 499 # not want to add a newline as `process_output` has stuff to add.
500 500 # The difficulty is that `process_input` doesn't know if
501 501 # `process_output` will be called---so it doesn't know if there is
502 502 # Out[] in the code block. The requires that we include a hack in
503 503 # `process_block`. See the comments there.
504 504 #
505 505 ret.append(processed_output)
506 506 elif is_semicolon:
507 507 # Make sure there is a newline after the semicolon.
508 508 ret.append('')
509 509
510 510 # context information
511 511 filename = "Unknown"
512 512 lineno = 0
513 513 if self.directive.state:
514 514 filename = self.directive.state.document.current_source
515 515 lineno = self.directive.state.document.current_line
516 516
517 517 # output any exceptions raised during execution to stdout
518 518 # unless :okexcept: has been specified.
519 519 if not is_okexcept and "Traceback" in processed_output:
520 520 s = "\nException in %s at block ending on line %s\n" % (filename, lineno)
521 521 s += "Specify :okexcept: as an option in the ipython:: block to suppress this message\n"
522 522 sys.stdout.write('\n\n>>>' + ('-' * 73))
523 523 sys.stdout.write(s)
524 524 sys.stdout.write(processed_output)
525 525 sys.stdout.write('<<<' + ('-' * 73) + '\n\n')
526 526
527 527 # output any warning raised during execution to stdout
528 528 # unless :okwarning: has been specified.
529 529 if not is_okwarning:
530 530 for w in ws:
531 531 s = "\nWarning in %s at block ending on line %s\n" % (filename, lineno)
532 532 s += "Specify :okwarning: as an option in the ipython:: block to suppress this message\n"
533 533 sys.stdout.write('\n\n>>>' + ('-' * 73))
534 534 sys.stdout.write(s)
535 535 sys.stdout.write(('-' * 76) + '\n')
536 536 s=warnings.formatwarning(w.message, w.category,
537 537 w.filename, w.lineno, w.line)
538 538 sys.stdout.write(s)
539 539 sys.stdout.write('<<<' + ('-' * 73) + '\n')
540 540
541 541 self.cout.truncate(0)
542 542
543 543 return (ret, input_lines, processed_output,
544 544 is_doctest, decorator, image_file, image_directive)
545 545
546 546
547 547 def process_output(self, data, output_prompt, input_lines, output,
548 548 is_doctest, decorator, image_file):
549 549 """
550 550 Process data block for OUTPUT token.
551 551
552 552 """
553 553 # Recall: `data` is the submitted output, and `output` is the processed
554 554 # output from `input_lines`.
555 555
556 556 TAB = ' ' * 4
557 557
558 558 if is_doctest and output is not None:
559 559
560 560 found = output # This is the processed output
561 561 found = found.strip()
562 562 submitted = data.strip()
563 563
564 564 if self.directive is None:
565 565 source = 'Unavailable'
566 566 content = 'Unavailable'
567 567 else:
568 568 source = self.directive.state.document.current_source
569 569 content = self.directive.content
570 570 # Add tabs and join into a single string.
571 571 content = '\n'.join([TAB + line for line in content])
572 572
573 573 # Make sure the output contains the output prompt.
574 574 ind = found.find(output_prompt)
575 575 if ind < 0:
576 576 e = ('output does not contain output prompt\n\n'
577 577 'Document source: {0}\n\n'
578 578 'Raw content: \n{1}\n\n'
579 579 'Input line(s):\n{TAB}{2}\n\n'
580 580 'Output line(s):\n{TAB}{3}\n\n')
581 581 e = e.format(source, content, '\n'.join(input_lines),
582 582 repr(found), TAB=TAB)
583 583 raise RuntimeError(e)
584 584 found = found[len(output_prompt):].strip()
585 585
586 586 # Handle the actual doctest comparison.
587 587 if decorator.strip() == '@doctest':
588 588 # Standard doctest
589 589 if found != submitted:
590 590 e = ('doctest failure\n\n'
591 591 'Document source: {0}\n\n'
592 592 'Raw content: \n{1}\n\n'
593 593 'On input line(s):\n{TAB}{2}\n\n'
594 594 'we found output:\n{TAB}{3}\n\n'
595 595 'instead of the expected:\n{TAB}{4}\n\n')
596 596 e = e.format(source, content, '\n'.join(input_lines),
597 597 repr(found), repr(submitted), TAB=TAB)
598 598 raise RuntimeError(e)
599 599 else:
600 600 self.custom_doctest(decorator, input_lines, found, submitted)
601 601
602 602 # When in verbatim mode, this holds additional submitted output
603 603 # to be written in the final Sphinx output.
604 604 # https://github.com/ipython/ipython/issues/5776
605 605 out_data = []
606 606
607 607 is_verbatim = decorator=='@verbatim' or self.is_verbatim
608 608 if is_verbatim and data.strip():
609 609 # Note that `ret` in `process_block` has '' as its last element if
610 610 # the code block was in verbatim mode. So if there is no submitted
611 611 # output, then we will have proper spacing only if we do not add
612 612 # an additional '' to `out_data`. This is why we condition on
613 613 # `and data.strip()`.
614 614
615 615 # The submitted output has no output prompt. If we want the
616 616 # prompt and the code to appear, we need to join them now
617 617 # instead of adding them separately---as this would create an
618 618 # undesired newline. How we do this ultimately depends on the
619 619 # format of the output regex. I'll do what works for the default
620 620 # prompt for now, and we might have to adjust if it doesn't work
621 621 # in other cases. Finally, the submitted output does not have
622 622 # a trailing newline, so we must add it manually.
623 623 out_data.append("{0} {1}\n".format(output_prompt, data))
624 624
625 625 return out_data
626 626
627 627 def process_comment(self, data):
628 628 """Process data fPblock for COMMENT token."""
629 629 if not self.is_suppress:
630 630 return [data]
631 631
632 632 def save_image(self, image_file):
633 633 """
634 634 Saves the image file to disk.
635 635 """
636 636 self.ensure_pyplot()
637 637 command = 'plt.gcf().savefig("%s")'%image_file
638 638 #print 'SAVEFIG', command # dbg
639 639 self.process_input_line('bookmark ipy_thisdir', store_history=False)
640 640 self.process_input_line('cd -b ipy_savedir', store_history=False)
641 641 self.process_input_line(command, store_history=False)
642 642 self.process_input_line('cd -b ipy_thisdir', store_history=False)
643 643 self.process_input_line('bookmark -d ipy_thisdir', store_history=False)
644 644 self.clear_cout()
645 645
646 646 def process_block(self, block):
647 647 """
648 648 process block from the block_parser and return a list of processed lines
649 649 """
650 650 ret = []
651 651 output = None
652 652 input_lines = None
653 653 lineno = self.IP.execution_count
654 654
655 655 input_prompt = self.promptin % lineno
656 656 output_prompt = self.promptout % lineno
657 657 image_file = None
658 658 image_directive = None
659 659
660 found_input = False
660 661 for token, data in block:
661 662 if token == COMMENT:
662 663 out_data = self.process_comment(data)
663 664 elif token == INPUT:
665 found_input = True
664 666 (out_data, input_lines, output, is_doctest,
665 667 decorator, image_file, image_directive) = \
666 668 self.process_input(data, input_prompt, lineno)
667 669 elif token == OUTPUT:
670 if not found_input:
671
672 TAB = ' ' * 4
673 linenumber = 0
674 source = 'Unavailable'
675 content = 'Unavailable'
676 if self.directive:
677 linenumber = self.directive.state.document.current_line
678 source = self.directive.state.document.current_source
679 content = self.directive.content
680 # Add tabs and join into a single string.
681 content = '\n'.join([TAB + line for line in content])
682
683 e = ('\n\nInvalid block: Block contains an output prompt '
684 'without an input prompt.\n\n'
685 'Document source: {0}\n\n'
686 'Content begins at line {1}: \n\n{2}\n\n'
687 'Problematic block within content: \n\n{TAB}{3}\n\n')
688 e = e.format(source, linenumber, content, block, TAB=TAB)
689
690 # Write, rather than include in exception, since Sphinx
691 # will truncate tracebacks.
692 sys.stdout.write(e)
693 raise RuntimeError('An invalid block was detected.')
694
668 695 out_data = \
669 696 self.process_output(data, output_prompt, input_lines,
670 697 output, is_doctest, decorator,
671 698 image_file)
672 699 if out_data:
673 700 # Then there was user submitted output in verbatim mode.
674 701 # We need to remove the last element of `ret` that was
675 702 # added in `process_input`, as it is '' and would introduce
676 703 # an undesirable newline.
677 704 assert(ret[-1] == '')
678 705 del ret[-1]
679 706
680 707 if out_data:
681 708 ret.extend(out_data)
682 709
683 710 # save the image files
684 711 if image_file is not None:
685 712 self.save_image(image_file)
686 713
687 714 return ret, image_directive
688 715
689 716 def ensure_pyplot(self):
690 717 """
691 718 Ensures that pyplot has been imported into the embedded IPython shell.
692 719
693 720 Also, makes sure to set the backend appropriately if not set already.
694 721
695 722 """
696 723 # We are here if the @figure pseudo decorator was used. Thus, it's
697 724 # possible that we could be here even if python_mplbackend were set to
698 725 # `None`. That's also strange and perhaps worthy of raising an
699 726 # exception, but for now, we just set the backend to 'agg'.
700 727
701 728 if not self._pyplot_imported:
702 729 if 'matplotlib.backends' not in sys.modules:
703 730 # Then ipython_matplotlib was set to None but there was a
704 731 # call to the @figure decorator (and ipython_execlines did
705 732 # not set a backend).
706 733 #raise Exception("No backend was set, but @figure was used!")
707 734 import matplotlib
708 735 matplotlib.use('agg')
709 736
710 737 # Always import pyplot into embedded shell.
711 738 self.process_input_line('import matplotlib.pyplot as plt',
712 739 store_history=False)
713 740 self._pyplot_imported = True
714 741
715 742 def process_pure_python(self, content):
716 743 """
717 744 content is a list of strings. it is unedited directive content
718 745
719 746 This runs it line by line in the InteractiveShell, prepends
720 747 prompts as needed capturing stderr and stdout, then returns
721 748 the content as a list as if it were ipython code
722 749 """
723 750 output = []
724 751 savefig = False # keep up with this to clear figure
725 752 multiline = False # to handle line continuation
726 753 multiline_start = None
727 754 fmtin = self.promptin
728 755
729 756 ct = 0
730 757
731 758 for lineno, line in enumerate(content):
732 759
733 760 line_stripped = line.strip()
734 761 if not len(line):
735 762 output.append(line)
736 763 continue
737 764
738 765 # handle decorators
739 766 if line_stripped.startswith('@'):
740 767 output.extend([line])
741 768 if 'savefig' in line:
742 769 savefig = True # and need to clear figure
743 770 continue
744 771
745 772 # handle comments
746 773 if line_stripped.startswith('#'):
747 774 output.extend([line])
748 775 continue
749 776
750 777 # deal with lines checking for multiline
751 778 continuation = u' %s:'% ''.join(['.']*(len(str(ct))+2))
752 779 if not multiline:
753 780 modified = u"%s %s" % (fmtin % ct, line_stripped)
754 781 output.append(modified)
755 782 ct += 1
756 783 try:
757 784 ast.parse(line_stripped)
758 785 output.append(u'')
759 786 except Exception: # on a multiline
760 787 multiline = True
761 788 multiline_start = lineno
762 789 else: # still on a multiline
763 790 modified = u'%s %s' % (continuation, line)
764 791 output.append(modified)
765 792
766 793 # if the next line is indented, it should be part of multiline
767 794 if len(content) > lineno + 1:
768 795 nextline = content[lineno + 1]
769 796 if len(nextline) - len(nextline.lstrip()) > 3:
770 797 continue
771 798 try:
772 799 mod = ast.parse(
773 800 '\n'.join(content[multiline_start:lineno+1]))
774 801 if isinstance(mod.body[0], ast.FunctionDef):
775 802 # check to see if we have the whole function
776 803 for element in mod.body[0].body:
777 804 if isinstance(element, ast.Return):
778 805 multiline = False
779 806 else:
780 807 output.append(u'')
781 808 multiline = False
782 809 except Exception:
783 810 pass
784 811
785 812 if savefig: # clear figure if plotted
786 813 self.ensure_pyplot()
787 814 self.process_input_line('plt.clf()', store_history=False)
788 815 self.clear_cout()
789 816 savefig = False
790 817
791 818 return output
792 819
793 820 def custom_doctest(self, decorator, input_lines, found, submitted):
794 821 """
795 822 Perform a specialized doctest.
796 823
797 824 """
798 825 from .custom_doctests import doctests
799 826
800 827 args = decorator.split()
801 828 doctest_type = args[1]
802 829 if doctest_type in doctests:
803 830 doctests[doctest_type](self, args, input_lines, found, submitted)
804 831 else:
805 832 e = "Invalid option to @doctest: {0}".format(doctest_type)
806 833 raise Exception(e)
807 834
808 835
809 836 class IPythonDirective(Directive):
810 837
811 838 has_content = True
812 839 required_arguments = 0
813 840 optional_arguments = 4 # python, suppress, verbatim, doctest
814 841 final_argumuent_whitespace = True
815 842 option_spec = { 'python': directives.unchanged,
816 843 'suppress' : directives.flag,
817 844 'verbatim' : directives.flag,
818 845 'doctest' : directives.flag,
819 846 'okexcept': directives.flag,
820 847 'okwarning': directives.flag
821 848 }
822 849
823 850 shell = None
824 851
825 852 seen_docs = set()
826 853
827 854 def get_config_options(self):
828 855 # contains sphinx configuration variables
829 856 config = self.state.document.settings.env.config
830 857
831 858 # get config variables to set figure output directory
832 859 confdir = self.state.document.settings.env.app.confdir
833 860 savefig_dir = config.ipython_savefig_dir
834 861 source_dir = os.path.dirname(self.state.document.current_source)
835 862 if savefig_dir is None:
836 863 savefig_dir = config.html_static_path
837 864 if isinstance(savefig_dir, list):
838 865 savefig_dir = savefig_dir[0] # safe to assume only one path?
839 866 savefig_dir = os.path.join(confdir, savefig_dir)
840 867
841 868 # get regex and prompt stuff
842 869 rgxin = config.ipython_rgxin
843 870 rgxout = config.ipython_rgxout
844 871 promptin = config.ipython_promptin
845 872 promptout = config.ipython_promptout
846 873 mplbackend = config.ipython_mplbackend
847 874 exec_lines = config.ipython_execlines
848 875 hold_count = config.ipython_holdcount
849 876
850 877 return (savefig_dir, source_dir, rgxin, rgxout,
851 878 promptin, promptout, mplbackend, exec_lines, hold_count)
852 879
853 880 def setup(self):
854 881 # Get configuration values.
855 882 (savefig_dir, source_dir, rgxin, rgxout, promptin, promptout,
856 883 mplbackend, exec_lines, hold_count) = self.get_config_options()
857 884
858 885 if self.shell is None:
859 886 # We will be here many times. However, when the
860 887 # EmbeddedSphinxShell is created, its interactive shell member
861 888 # is the same for each instance.
862 889
863 890 if mplbackend:
864 891 import matplotlib
865 892 # Repeated calls to use() will not hurt us since `mplbackend`
866 893 # is the same each time.
867 894 matplotlib.use(mplbackend)
868 895
869 896 # Must be called after (potentially) importing matplotlib and
870 897 # setting its backend since exec_lines might import pylab.
871 898 self.shell = EmbeddedSphinxShell(exec_lines)
872 899
873 900 # Store IPython directive to enable better error messages
874 901 self.shell.directive = self
875 902
876 903 # reset the execution count if we haven't processed this doc
877 904 #NOTE: this may be borked if there are multiple seen_doc tmp files
878 905 #check time stamp?
879 906 if not self.state.document.current_source in self.seen_docs:
880 907 self.shell.IP.history_manager.reset()
881 908 self.shell.IP.execution_count = 1
882 909 self.shell.IP.prompt_manager.width = 0
883 910 self.seen_docs.add(self.state.document.current_source)
884 911
885 912 # and attach to shell so we don't have to pass them around
886 913 self.shell.rgxin = rgxin
887 914 self.shell.rgxout = rgxout
888 915 self.shell.promptin = promptin
889 916 self.shell.promptout = promptout
890 917 self.shell.savefig_dir = savefig_dir
891 918 self.shell.source_dir = source_dir
892 919 self.shell.hold_count = hold_count
893 920
894 921 # setup bookmark for saving figures directory
895 922 self.shell.process_input_line('bookmark ipy_savedir %s'%savefig_dir,
896 923 store_history=False)
897 924 self.shell.clear_cout()
898 925
899 926 return rgxin, rgxout, promptin, promptout
900 927
901 928 def teardown(self):
902 929 # delete last bookmark
903 930 self.shell.process_input_line('bookmark -d ipy_savedir',
904 931 store_history=False)
905 932 self.shell.clear_cout()
906 933
907 934 def run(self):
908 935 debug = False
909 936
910 937 #TODO, any reason block_parser can't be a method of embeddable shell
911 938 # then we wouldn't have to carry these around
912 939 rgxin, rgxout, promptin, promptout = self.setup()
913 940
914 941 options = self.options
915 942 self.shell.is_suppress = 'suppress' in options
916 943 self.shell.is_doctest = 'doctest' in options
917 944 self.shell.is_verbatim = 'verbatim' in options
918 945 self.shell.is_okexcept = 'okexcept' in options
919 946 self.shell.is_okwarning = 'okwarning' in options
920 947
921 948 # handle pure python code
922 949 if 'python' in self.arguments:
923 950 content = self.content
924 951 self.content = self.shell.process_pure_python(content)
925 952
953 # parts consists of all text within the ipython-block.
954 # Each part is an input/output block.
926 955 parts = '\n'.join(self.content).split('\n\n')
927 956
928 957 lines = ['.. code-block:: ipython', '']
929 958 figures = []
930 959
931 960 for part in parts:
932 961 block = block_parser(part, rgxin, rgxout, promptin, promptout)
933 962 if len(block):
934 963 rows, figure = self.shell.process_block(block)
935 964 for row in rows:
936 965 lines.extend([' {0}'.format(line)
937 966 for line in row.split('\n')])
938 967
939 968 if figure is not None:
940 969 figures.append(figure)
941 970
942 971 for figure in figures:
943 972 lines.append('')
944 973 lines.extend(figure.split('\n'))
945 974 lines.append('')
946 975
947 976 if len(lines) > 2:
948 977 if debug:
949 978 print('\n'.join(lines))
950 979 else:
951 980 # This has to do with input, not output. But if we comment
952 981 # these lines out, then no IPython code will appear in the
953 982 # final output.
954 983 self.state_machine.insert_input(
955 984 lines, self.state_machine.input_lines.source(0))
956 985
957 986 # cleanup
958 987 self.teardown()
959 988
960 989 return []
961 990
962 991 # Enable as a proper Sphinx directive
963 992 def setup(app):
964 993 setup.app = app
965 994
966 995 app.add_directive('ipython', IPythonDirective)
967 996 app.add_config_value('ipython_savefig_dir', None, 'env')
968 997 app.add_config_value('ipython_rgxin',
969 998 re.compile('In \[(\d+)\]:\s?(.*)\s*'), 'env')
970 999 app.add_config_value('ipython_rgxout',
971 1000 re.compile('Out\[(\d+)\]:\s?(.*)\s*'), 'env')
972 1001 app.add_config_value('ipython_promptin', 'In [%d]:', 'env')
973 1002 app.add_config_value('ipython_promptout', 'Out[%d]:', 'env')
974 1003
975 1004 # We could just let matplotlib pick whatever is specified as the default
976 1005 # backend in the matplotlibrc file, but this would cause issues if the
977 1006 # backend didn't work in headless environments. For this reason, 'agg'
978 1007 # is a good default backend choice.
979 1008 app.add_config_value('ipython_mplbackend', 'agg', 'env')
980 1009
981 1010 # If the user sets this config value to `None`, then EmbeddedSphinxShell's
982 1011 # __init__ method will treat it as [].
983 1012 execlines = ['import numpy as np', 'import matplotlib.pyplot as plt']
984 1013 app.add_config_value('ipython_execlines', execlines, 'env')
985 1014
986 1015 app.add_config_value('ipython_holdcount', True, 'env')
987 1016
988 1017 # Simple smoke test, needs to be converted to a proper automatic test.
989 1018 def test():
990 1019
991 1020 examples = [
992 1021 r"""
993 1022 In [9]: pwd
994 1023 Out[9]: '/home/jdhunter/py4science/book'
995 1024
996 1025 In [10]: cd bookdata/
997 1026 /home/jdhunter/py4science/book/bookdata
998 1027
999 1028 In [2]: from pylab import *
1000 1029
1001 1030 In [2]: ion()
1002 1031
1003 1032 In [3]: im = imread('stinkbug.png')
1004 1033
1005 1034 @savefig mystinkbug.png width=4in
1006 1035 In [4]: imshow(im)
1007 1036 Out[4]: <matplotlib.image.AxesImage object at 0x39ea850>
1008 1037
1009 1038 """,
1010 1039 r"""
1011 1040
1012 1041 In [1]: x = 'hello world'
1013 1042
1014 1043 # string methods can be
1015 1044 # used to alter the string
1016 1045 @doctest
1017 1046 In [2]: x.upper()
1018 1047 Out[2]: 'HELLO WORLD'
1019 1048
1020 1049 @verbatim
1021 1050 In [3]: x.st<TAB>
1022 1051 x.startswith x.strip
1023 1052 """,
1024 1053 r"""
1025 1054
1026 1055 In [130]: url = 'http://ichart.finance.yahoo.com/table.csv?s=CROX\
1027 1056 .....: &d=9&e=22&f=2009&g=d&a=1&br=8&c=2006&ignore=.csv'
1028 1057
1029 1058 In [131]: print url.split('&')
1030 1059 ['http://ichart.finance.yahoo.com/table.csv?s=CROX', 'd=9', 'e=22', 'f=2009', 'g=d', 'a=1', 'b=8', 'c=2006', 'ignore=.csv']
1031 1060
1032 1061 In [60]: import urllib
1033 1062
1034 1063 """,
1035 1064 r"""\
1036 1065
1037 1066 In [133]: import numpy.random
1038 1067
1039 1068 @suppress
1040 1069 In [134]: numpy.random.seed(2358)
1041 1070
1042 1071 @doctest
1043 1072 In [135]: numpy.random.rand(10,2)
1044 1073 Out[135]:
1045 1074 array([[ 0.64524308, 0.59943846],
1046 1075 [ 0.47102322, 0.8715456 ],
1047 1076 [ 0.29370834, 0.74776844],
1048 1077 [ 0.99539577, 0.1313423 ],
1049 1078 [ 0.16250302, 0.21103583],
1050 1079 [ 0.81626524, 0.1312433 ],
1051 1080 [ 0.67338089, 0.72302393],
1052 1081 [ 0.7566368 , 0.07033696],
1053 1082 [ 0.22591016, 0.77731835],
1054 1083 [ 0.0072729 , 0.34273127]])
1055 1084
1056 1085 """,
1057 1086
1058 1087 r"""
1059 1088 In [106]: print x
1060 1089 jdh
1061 1090
1062 1091 In [109]: for i in range(10):
1063 1092 .....: print i
1064 1093 .....:
1065 1094 .....:
1066 1095 0
1067 1096 1
1068 1097 2
1069 1098 3
1070 1099 4
1071 1100 5
1072 1101 6
1073 1102 7
1074 1103 8
1075 1104 9
1076 1105 """,
1077 1106
1078 1107 r"""
1079 1108
1080 1109 In [144]: from pylab import *
1081 1110
1082 1111 In [145]: ion()
1083 1112
1084 1113 # use a semicolon to suppress the output
1085 1114 @savefig test_hist.png width=4in
1086 1115 In [151]: hist(np.random.randn(10000), 100);
1087 1116
1088 1117
1089 1118 @savefig test_plot.png width=4in
1090 1119 In [151]: plot(np.random.randn(10000), 'o');
1091 1120 """,
1092 1121
1093 1122 r"""
1094 1123 # use a semicolon to suppress the output
1095 1124 In [151]: plt.clf()
1096 1125
1097 1126 @savefig plot_simple.png width=4in
1098 1127 In [151]: plot([1,2,3])
1099 1128
1100 1129 @savefig hist_simple.png width=4in
1101 1130 In [151]: hist(np.random.randn(10000), 100);
1102 1131
1103 1132 """,
1104 1133 r"""
1105 1134 # update the current fig
1106 1135 In [151]: ylabel('number')
1107 1136
1108 1137 In [152]: title('normal distribution')
1109 1138
1110 1139
1111 1140 @savefig hist_with_text.png
1112 1141 In [153]: grid(True)
1113 1142
1114 1143 @doctest float
1115 1144 In [154]: 0.1 + 0.2
1116 1145 Out[154]: 0.3
1117 1146
1118 1147 @doctest float
1119 1148 In [155]: np.arange(16).reshape(4,4)
1120 1149 Out[155]:
1121 1150 array([[ 0, 1, 2, 3],
1122 1151 [ 4, 5, 6, 7],
1123 1152 [ 8, 9, 10, 11],
1124 1153 [12, 13, 14, 15]])
1125 1154
1126 1155 In [1]: x = np.arange(16, dtype=float).reshape(4,4)
1127 1156
1128 1157 In [2]: x[0,0] = np.inf
1129 1158
1130 1159 In [3]: x[0,1] = np.nan
1131 1160
1132 1161 @doctest float
1133 1162 In [4]: x
1134 1163 Out[4]:
1135 1164 array([[ inf, nan, 2., 3.],
1136 1165 [ 4., 5., 6., 7.],
1137 1166 [ 8., 9., 10., 11.],
1138 1167 [ 12., 13., 14., 15.]])
1139 1168
1140 1169
1141 1170 """,
1142 1171 ]
1143 1172 # skip local-file depending first example:
1144 1173 examples = examples[1:]
1145 1174
1146 1175 #ipython_directive.DEBUG = True # dbg
1147 1176 #options = dict(suppress=True) # dbg
1148 1177 options = dict()
1149 1178 for example in examples:
1150 1179 content = example.split('\n')
1151 1180 IPythonDirective('debug', arguments=None, options=options,
1152 1181 content=content, lineno=0,
1153 1182 content_offset=None, block_text=None,
1154 1183 state=None, state_machine=None,
1155 1184 )
1156 1185
1157 1186 # Run test suite as a script
1158 1187 if __name__=='__main__':
1159 1188 if not os.path.isdir('_static'):
1160 1189 os.mkdir('_static')
1161 1190 test()
1162 1191 print('All OK? Check figures in _static/')
General Comments 0
You need to be logged in to leave comments. Login now