##// END OF EJS Templates
import StringIO from io package
Srinivas Reddy Thatiparthy -
Show More
@@ -1,1183 +1,1177 b''
1 1 # -*- coding: utf-8 -*-
2 2 """
3 3 Sphinx directive to support embedded IPython code.
4 4
5 5 This directive allows pasting of entire interactive IPython sessions, prompts
6 6 and all, and their code will actually get re-executed at doc build time, with
7 7 all prompts renumbered sequentially. It also allows you to input code as a pure
8 8 python input by giving the argument python to the directive. The output looks
9 9 like an interactive ipython section.
10 10
11 11 To enable this directive, simply list it in your Sphinx ``conf.py`` file
12 12 (making sure the directory where you placed it is visible to sphinx, as is
13 13 needed for all Sphinx directives). For example, to enable syntax highlighting
14 14 and the IPython directive::
15 15
16 16 extensions = ['IPython.sphinxext.ipython_console_highlighting',
17 17 'IPython.sphinxext.ipython_directive']
18 18
19 19 The IPython directive outputs code-blocks with the language 'ipython'. So
20 20 if you do not have the syntax highlighting extension enabled as well, then
21 21 all rendered code-blocks will be uncolored. By default this directive assumes
22 22 that your prompts are unchanged IPython ones, but this can be customized.
23 23 The configurable options that can be placed in conf.py are:
24 24
25 25 ipython_savefig_dir:
26 26 The directory in which to save the figures. This is relative to the
27 27 Sphinx source directory. The default is `html_static_path`.
28 28 ipython_rgxin:
29 29 The compiled regular expression to denote the start of IPython input
30 30 lines. The default is re.compile('In \[(\d+)\]:\s?(.*)\s*'). You
31 31 shouldn't need to change this.
32 32 ipython_rgxout:
33 33 The compiled regular expression to denote the start of IPython output
34 34 lines. The default is re.compile('Out\[(\d+)\]:\s?(.*)\s*'). You
35 35 shouldn't need to change this.
36 36 ipython_promptin:
37 37 The string to represent the IPython input prompt in the generated ReST.
38 38 The default is 'In [%d]:'. This expects that the line numbers are used
39 39 in the prompt.
40 40 ipython_promptout:
41 41 The string to represent the IPython prompt in the generated ReST. The
42 42 default is 'Out [%d]:'. This expects that the line numbers are used
43 43 in the prompt.
44 44 ipython_mplbackend:
45 45 The string which specifies if the embedded Sphinx shell should import
46 46 Matplotlib and set the backend. The value specifies a backend that is
47 47 passed to `matplotlib.use()` before any lines in `ipython_execlines` are
48 48 executed. If not specified in conf.py, then the default value of 'agg' is
49 49 used. To use the IPython directive without matplotlib as a dependency, set
50 50 the value to `None`. It may end up that matplotlib is still imported
51 51 if the user specifies so in `ipython_execlines` or makes use of the
52 52 @savefig pseudo decorator.
53 53 ipython_execlines:
54 54 A list of strings to be exec'd in the embedded Sphinx shell. Typical
55 55 usage is to make certain packages always available. Set this to an empty
56 56 list if you wish to have no imports always available. If specified in
57 57 conf.py as `None`, then it has the effect of making no imports available.
58 58 If omitted from conf.py altogether, then the default value of
59 59 ['import numpy as np', 'import matplotlib.pyplot as plt'] is used.
60 60 ipython_holdcount
61 61 When the @suppress pseudo-decorator is used, the execution count can be
62 62 incremented or not. The default behavior is to hold the execution count,
63 63 corresponding to a value of `True`. Set this to `False` to increment
64 64 the execution count after each suppressed command.
65 65
66 66 As an example, to use the IPython directive when `matplotlib` is not available,
67 67 one sets the backend to `None`::
68 68
69 69 ipython_mplbackend = None
70 70
71 71 An example usage of the directive is:
72 72
73 73 .. code-block:: rst
74 74
75 75 .. ipython::
76 76
77 77 In [1]: x = 1
78 78
79 79 In [2]: y = x**2
80 80
81 81 In [3]: print(y)
82 82
83 83 See http://matplotlib.org/sampledoc/ipython_directive.html for additional
84 84 documentation.
85 85
86 86 Pseudo-Decorators
87 87 =================
88 88
89 89 Note: Only one decorator is supported per input. If more than one decorator
90 90 is specified, then only the last one is used.
91 91
92 92 In addition to the Pseudo-Decorators/options described at the above link,
93 93 several enhancements have been made. The directive will emit a message to the
94 94 console at build-time if code-execution resulted in an exception or warning.
95 95 You can suppress these on a per-block basis by specifying the :okexcept:
96 96 or :okwarning: options:
97 97
98 98 .. code-block:: rst
99 99
100 100 .. ipython::
101 101 :okexcept:
102 102 :okwarning:
103 103
104 104 In [1]: 1/0
105 105 In [2]: # raise warning.
106 106
107 107 ToDo
108 108 ----
109 109
110 110 - Turn the ad-hoc test() function into a real test suite.
111 111 - Break up ipython-specific functionality from matplotlib stuff into better
112 112 separated code.
113 113
114 114 Authors
115 115 -------
116 116
117 117 - John D Hunter: orignal author.
118 118 - Fernando Perez: refactoring, documentation, cleanups, port to 0.11.
119 119 - VΓ‘clavΕ milauer <eudoxos-AT-arcig.cz>: Prompt generalizations.
120 120 - Skipper Seabold, refactoring, cleanups, pure python addition
121 121 """
122 122
123 123 #-----------------------------------------------------------------------------
124 124 # Imports
125 125 #-----------------------------------------------------------------------------
126 126
127 127 # Stdlib
128 128 import atexit
129 129 import os
130 130 import re
131 131 import sys
132 132 import tempfile
133 133 import ast
134 134 import warnings
135 135 import shutil
136
136 from io import StringIO
137 137
138 138 # Third-party
139 139 from docutils.parsers.rst import directives
140 140 from sphinx.util.compat import Directive
141 141
142 142 # Our own
143 143 from traitlets.config import Config
144 144 from IPython import InteractiveShell
145 145 from IPython.core.profiledir import ProfileDir
146 146 from IPython.utils import io
147 from IPython.utils.py3compat import PY3
148
149 if PY3:
150 from io import StringIO
151 else:
152 from StringIO import StringIO
153 147
154 148 #-----------------------------------------------------------------------------
155 149 # Globals
156 150 #-----------------------------------------------------------------------------
157 151 # for tokenizing blocks
158 152 COMMENT, INPUT, OUTPUT = range(3)
159 153
160 154 #-----------------------------------------------------------------------------
161 155 # Functions and class declarations
162 156 #-----------------------------------------------------------------------------
163 157
164 158 def block_parser(part, rgxin, rgxout, fmtin, fmtout):
165 159 """
166 160 part is a string of ipython text, comprised of at most one
167 161 input, one output, comments, and blank lines. The block parser
168 162 parses the text into a list of::
169 163
170 164 blocks = [ (TOKEN0, data0), (TOKEN1, data1), ...]
171 165
172 166 where TOKEN is one of [COMMENT | INPUT | OUTPUT ] and
173 167 data is, depending on the type of token::
174 168
175 169 COMMENT : the comment string
176 170
177 171 INPUT: the (DECORATOR, INPUT_LINE, REST) where
178 172 DECORATOR: the input decorator (or None)
179 173 INPUT_LINE: the input as string (possibly multi-line)
180 174 REST : any stdout generated by the input line (not OUTPUT)
181 175
182 176 OUTPUT: the output string, possibly multi-line
183 177
184 178 """
185 179 block = []
186 180 lines = part.split('\n')
187 181 N = len(lines)
188 182 i = 0
189 183 decorator = None
190 184 while 1:
191 185
192 186 if i==N:
193 187 # nothing left to parse -- the last line
194 188 break
195 189
196 190 line = lines[i]
197 191 i += 1
198 192 line_stripped = line.strip()
199 193 if line_stripped.startswith('#'):
200 194 block.append((COMMENT, line))
201 195 continue
202 196
203 197 if line_stripped.startswith('@'):
204 198 # Here is where we assume there is, at most, one decorator.
205 199 # Might need to rethink this.
206 200 decorator = line_stripped
207 201 continue
208 202
209 203 # does this look like an input line?
210 204 matchin = rgxin.match(line)
211 205 if matchin:
212 206 lineno, inputline = int(matchin.group(1)), matchin.group(2)
213 207
214 208 # the ....: continuation string
215 209 continuation = ' %s:'%''.join(['.']*(len(str(lineno))+2))
216 210 Nc = len(continuation)
217 211 # input lines can continue on for more than one line, if
218 212 # we have a '\' line continuation char or a function call
219 213 # echo line 'print'. The input line can only be
220 214 # terminated by the end of the block or an output line, so
221 215 # we parse out the rest of the input line if it is
222 216 # multiline as well as any echo text
223 217
224 218 rest = []
225 219 while i<N:
226 220
227 221 # look ahead; if the next line is blank, or a comment, or
228 222 # an output line, we're done
229 223
230 224 nextline = lines[i]
231 225 matchout = rgxout.match(nextline)
232 226 #print "nextline=%s, continuation=%s, starts=%s"%(nextline, continuation, nextline.startswith(continuation))
233 227 if matchout or nextline.startswith('#'):
234 228 break
235 229 elif nextline.startswith(continuation):
236 230 # The default ipython_rgx* treat the space following the colon as optional.
237 231 # However, If the space is there we must consume it or code
238 232 # employing the cython_magic extension will fail to execute.
239 233 #
240 234 # This works with the default ipython_rgx* patterns,
241 235 # If you modify them, YMMV.
242 236 nextline = nextline[Nc:]
243 237 if nextline and nextline[0] == ' ':
244 238 nextline = nextline[1:]
245 239
246 240 inputline += '\n' + nextline
247 241 else:
248 242 rest.append(nextline)
249 243 i+= 1
250 244
251 245 block.append((INPUT, (decorator, inputline, '\n'.join(rest))))
252 246 continue
253 247
254 248 # if it looks like an output line grab all the text to the end
255 249 # of the block
256 250 matchout = rgxout.match(line)
257 251 if matchout:
258 252 lineno, output = int(matchout.group(1)), matchout.group(2)
259 253 if i<N-1:
260 254 output = '\n'.join([output] + lines[i:])
261 255
262 256 block.append((OUTPUT, output))
263 257 break
264 258
265 259 return block
266 260
267 261
268 262 class EmbeddedSphinxShell(object):
269 263 """An embedded IPython instance to run inside Sphinx"""
270 264
271 265 def __init__(self, exec_lines=None):
272 266
273 267 self.cout = StringIO()
274 268
275 269 if exec_lines is None:
276 270 exec_lines = []
277 271
278 272 # Create config object for IPython
279 273 config = Config()
280 274 config.HistoryManager.hist_file = ':memory:'
281 275 config.InteractiveShell.autocall = False
282 276 config.InteractiveShell.autoindent = False
283 277 config.InteractiveShell.colors = 'NoColor'
284 278
285 279 # create a profile so instance history isn't saved
286 280 tmp_profile_dir = tempfile.mkdtemp(prefix='profile_')
287 281 profname = 'auto_profile_sphinx_build'
288 282 pdir = os.path.join(tmp_profile_dir,profname)
289 283 profile = ProfileDir.create_profile_dir(pdir)
290 284
291 285 # Create and initialize global ipython, but don't start its mainloop.
292 286 # This will persist across different EmbededSphinxShell instances.
293 287 IP = InteractiveShell.instance(config=config, profile_dir=profile)
294 288 atexit.register(self.cleanup)
295 289
296 290 sys.stdout = self.cout
297 291 sys.stderr = self.cout
298 292
299 293 # For debugging, so we can see normal output, use this:
300 294 #from IPython.utils.io import Tee
301 295 #sys.stdout = Tee(self.cout, channel='stdout') # dbg
302 296 #sys.stderr = Tee(self.cout, channel='stderr') # dbg
303 297
304 298 # Store a few parts of IPython we'll need.
305 299 self.IP = IP
306 300 self.user_ns = self.IP.user_ns
307 301 self.user_global_ns = self.IP.user_global_ns
308 302
309 303 self.input = ''
310 304 self.output = ''
311 305 self.tmp_profile_dir = tmp_profile_dir
312 306
313 307 self.is_verbatim = False
314 308 self.is_doctest = False
315 309 self.is_suppress = False
316 310
317 311 # Optionally, provide more detailed information to shell.
318 312 # this is assigned by the SetUp method of IPythonDirective
319 313 # to point at itself.
320 314 #
321 315 # So, you can access handy things at self.directive.state
322 316 self.directive = None
323 317
324 318 # on the first call to the savefig decorator, we'll import
325 319 # pyplot as plt so we can make a call to the plt.gcf().savefig
326 320 self._pyplot_imported = False
327 321
328 322 # Prepopulate the namespace.
329 323 for line in exec_lines:
330 324 self.process_input_line(line, store_history=False)
331 325
332 326 def cleanup(self):
333 327 shutil.rmtree(self.tmp_profile_dir, ignore_errors=True)
334 328
335 329 def clear_cout(self):
336 330 self.cout.seek(0)
337 331 self.cout.truncate(0)
338 332
339 333 def process_input_line(self, line, store_history=True):
340 334 """process the input, capturing stdout"""
341 335
342 336 stdout = sys.stdout
343 337 splitter = self.IP.input_splitter
344 338 try:
345 339 sys.stdout = self.cout
346 340 splitter.push(line)
347 341 more = splitter.push_accepts_more()
348 342 if not more:
349 343 source_raw = splitter.raw_reset()
350 344 self.IP.run_cell(source_raw, store_history=store_history)
351 345 finally:
352 346 sys.stdout = stdout
353 347
354 348 def process_image(self, decorator):
355 349 """
356 350 # build out an image directive like
357 351 # .. image:: somefile.png
358 352 # :width 4in
359 353 #
360 354 # from an input like
361 355 # savefig somefile.png width=4in
362 356 """
363 357 savefig_dir = self.savefig_dir
364 358 source_dir = self.source_dir
365 359 saveargs = decorator.split(' ')
366 360 filename = saveargs[1]
367 361 # insert relative path to image file in source
368 362 outfile = os.path.relpath(os.path.join(savefig_dir,filename),
369 363 source_dir)
370 364
371 365 imagerows = ['.. image:: %s'%outfile]
372 366
373 367 for kwarg in saveargs[2:]:
374 368 arg, val = kwarg.split('=')
375 369 arg = arg.strip()
376 370 val = val.strip()
377 371 imagerows.append(' :%s: %s'%(arg, val))
378 372
379 373 image_file = os.path.basename(outfile) # only return file name
380 374 image_directive = '\n'.join(imagerows)
381 375 return image_file, image_directive
382 376
383 377 # Callbacks for each type of token
384 378 def process_input(self, data, input_prompt, lineno):
385 379 """
386 380 Process data block for INPUT token.
387 381
388 382 """
389 383 decorator, input, rest = data
390 384 image_file = None
391 385 image_directive = None
392 386
393 387 is_verbatim = decorator=='@verbatim' or self.is_verbatim
394 388 is_doctest = (decorator is not None and \
395 389 decorator.startswith('@doctest')) or self.is_doctest
396 390 is_suppress = decorator=='@suppress' or self.is_suppress
397 391 is_okexcept = decorator=='@okexcept' or self.is_okexcept
398 392 is_okwarning = decorator=='@okwarning' or self.is_okwarning
399 393 is_savefig = decorator is not None and \
400 394 decorator.startswith('@savefig')
401 395
402 396 input_lines = input.split('\n')
403 397 if len(input_lines) > 1:
404 398 if input_lines[-1] != "":
405 399 input_lines.append('') # make sure there's a blank line
406 400 # so splitter buffer gets reset
407 401
408 402 continuation = ' %s:'%''.join(['.']*(len(str(lineno))+2))
409 403
410 404 if is_savefig:
411 405 image_file, image_directive = self.process_image(decorator)
412 406
413 407 ret = []
414 408 is_semicolon = False
415 409
416 410 # Hold the execution count, if requested to do so.
417 411 if is_suppress and self.hold_count:
418 412 store_history = False
419 413 else:
420 414 store_history = True
421 415
422 416 # Note: catch_warnings is not thread safe
423 417 with warnings.catch_warnings(record=True) as ws:
424 418 for i, line in enumerate(input_lines):
425 419 if line.endswith(';'):
426 420 is_semicolon = True
427 421
428 422 if i == 0:
429 423 # process the first input line
430 424 if is_verbatim:
431 425 self.process_input_line('')
432 426 self.IP.execution_count += 1 # increment it anyway
433 427 else:
434 428 # only submit the line in non-verbatim mode
435 429 self.process_input_line(line, store_history=store_history)
436 430 formatted_line = '%s %s'%(input_prompt, line)
437 431 else:
438 432 # process a continuation line
439 433 if not is_verbatim:
440 434 self.process_input_line(line, store_history=store_history)
441 435
442 436 formatted_line = '%s %s'%(continuation, line)
443 437
444 438 if not is_suppress:
445 439 ret.append(formatted_line)
446 440
447 441 if not is_suppress and len(rest.strip()) and is_verbatim:
448 442 # The "rest" is the standard output of the input. This needs to be
449 443 # added when in verbatim mode. If there is no "rest", then we don't
450 444 # add it, as the new line will be added by the processed output.
451 445 ret.append(rest)
452 446
453 447 # Fetch the processed output. (This is not the submitted output.)
454 448 self.cout.seek(0)
455 449 processed_output = self.cout.read()
456 450 if not is_suppress and not is_semicolon:
457 451 #
458 452 # In IPythonDirective.run, the elements of `ret` are eventually
459 453 # combined such that '' entries correspond to newlines. So if
460 454 # `processed_output` is equal to '', then the adding it to `ret`
461 455 # ensures that there is a blank line between consecutive inputs
462 456 # that have no outputs, as in:
463 457 #
464 458 # In [1]: x = 4
465 459 #
466 460 # In [2]: x = 5
467 461 #
468 462 # When there is processed output, it has a '\n' at the tail end. So
469 463 # adding the output to `ret` will provide the necessary spacing
470 464 # between consecutive input/output blocks, as in:
471 465 #
472 466 # In [1]: x
473 467 # Out[1]: 5
474 468 #
475 469 # In [2]: x
476 470 # Out[2]: 5
477 471 #
478 472 # When there is stdout from the input, it also has a '\n' at the
479 473 # tail end, and so this ensures proper spacing as well. E.g.:
480 474 #
481 475 # In [1]: print x
482 476 # 5
483 477 #
484 478 # In [2]: x = 5
485 479 #
486 480 # When in verbatim mode, `processed_output` is empty (because
487 481 # nothing was passed to IP. Sometimes the submitted code block has
488 482 # an Out[] portion and sometimes it does not. When it does not, we
489 483 # need to ensure proper spacing, so we have to add '' to `ret`.
490 484 # However, if there is an Out[] in the submitted code, then we do
491 485 # not want to add a newline as `process_output` has stuff to add.
492 486 # The difficulty is that `process_input` doesn't know if
493 487 # `process_output` will be called---so it doesn't know if there is
494 488 # Out[] in the code block. The requires that we include a hack in
495 489 # `process_block`. See the comments there.
496 490 #
497 491 ret.append(processed_output)
498 492 elif is_semicolon:
499 493 # Make sure there is a newline after the semicolon.
500 494 ret.append('')
501 495
502 496 # context information
503 497 filename = "Unknown"
504 498 lineno = 0
505 499 if self.directive.state:
506 500 filename = self.directive.state.document.current_source
507 501 lineno = self.directive.state.document.current_line
508 502
509 503 # output any exceptions raised during execution to stdout
510 504 # unless :okexcept: has been specified.
511 505 if not is_okexcept and "Traceback" in processed_output:
512 506 s = "\nException in %s at block ending on line %s\n" % (filename, lineno)
513 507 s += "Specify :okexcept: as an option in the ipython:: block to suppress this message\n"
514 508 sys.stdout.write('\n\n>>>' + ('-' * 73))
515 509 sys.stdout.write(s)
516 510 sys.stdout.write(processed_output)
517 511 sys.stdout.write('<<<' + ('-' * 73) + '\n\n')
518 512
519 513 # output any warning raised during execution to stdout
520 514 # unless :okwarning: has been specified.
521 515 if not is_okwarning:
522 516 for w in ws:
523 517 s = "\nWarning in %s at block ending on line %s\n" % (filename, lineno)
524 518 s += "Specify :okwarning: as an option in the ipython:: block to suppress this message\n"
525 519 sys.stdout.write('\n\n>>>' + ('-' * 73))
526 520 sys.stdout.write(s)
527 521 sys.stdout.write(('-' * 76) + '\n')
528 522 s=warnings.formatwarning(w.message, w.category,
529 523 w.filename, w.lineno, w.line)
530 524 sys.stdout.write(s)
531 525 sys.stdout.write('<<<' + ('-' * 73) + '\n')
532 526
533 527 self.cout.truncate(0)
534 528
535 529 return (ret, input_lines, processed_output,
536 530 is_doctest, decorator, image_file, image_directive)
537 531
538 532
539 533 def process_output(self, data, output_prompt, input_lines, output,
540 534 is_doctest, decorator, image_file):
541 535 """
542 536 Process data block for OUTPUT token.
543 537
544 538 """
545 539 # Recall: `data` is the submitted output, and `output` is the processed
546 540 # output from `input_lines`.
547 541
548 542 TAB = ' ' * 4
549 543
550 544 if is_doctest and output is not None:
551 545
552 546 found = output # This is the processed output
553 547 found = found.strip()
554 548 submitted = data.strip()
555 549
556 550 if self.directive is None:
557 551 source = 'Unavailable'
558 552 content = 'Unavailable'
559 553 else:
560 554 source = self.directive.state.document.current_source
561 555 content = self.directive.content
562 556 # Add tabs and join into a single string.
563 557 content = '\n'.join([TAB + line for line in content])
564 558
565 559 # Make sure the output contains the output prompt.
566 560 ind = found.find(output_prompt)
567 561 if ind < 0:
568 562 e = ('output does not contain output prompt\n\n'
569 563 'Document source: {0}\n\n'
570 564 'Raw content: \n{1}\n\n'
571 565 'Input line(s):\n{TAB}{2}\n\n'
572 566 'Output line(s):\n{TAB}{3}\n\n')
573 567 e = e.format(source, content, '\n'.join(input_lines),
574 568 repr(found), TAB=TAB)
575 569 raise RuntimeError(e)
576 570 found = found[len(output_prompt):].strip()
577 571
578 572 # Handle the actual doctest comparison.
579 573 if decorator.strip() == '@doctest':
580 574 # Standard doctest
581 575 if found != submitted:
582 576 e = ('doctest failure\n\n'
583 577 'Document source: {0}\n\n'
584 578 'Raw content: \n{1}\n\n'
585 579 'On input line(s):\n{TAB}{2}\n\n'
586 580 'we found output:\n{TAB}{3}\n\n'
587 581 'instead of the expected:\n{TAB}{4}\n\n')
588 582 e = e.format(source, content, '\n'.join(input_lines),
589 583 repr(found), repr(submitted), TAB=TAB)
590 584 raise RuntimeError(e)
591 585 else:
592 586 self.custom_doctest(decorator, input_lines, found, submitted)
593 587
594 588 # When in verbatim mode, this holds additional submitted output
595 589 # to be written in the final Sphinx output.
596 590 # https://github.com/ipython/ipython/issues/5776
597 591 out_data = []
598 592
599 593 is_verbatim = decorator=='@verbatim' or self.is_verbatim
600 594 if is_verbatim and data.strip():
601 595 # Note that `ret` in `process_block` has '' as its last element if
602 596 # the code block was in verbatim mode. So if there is no submitted
603 597 # output, then we will have proper spacing only if we do not add
604 598 # an additional '' to `out_data`. This is why we condition on
605 599 # `and data.strip()`.
606 600
607 601 # The submitted output has no output prompt. If we want the
608 602 # prompt and the code to appear, we need to join them now
609 603 # instead of adding them separately---as this would create an
610 604 # undesired newline. How we do this ultimately depends on the
611 605 # format of the output regex. I'll do what works for the default
612 606 # prompt for now, and we might have to adjust if it doesn't work
613 607 # in other cases. Finally, the submitted output does not have
614 608 # a trailing newline, so we must add it manually.
615 609 out_data.append("{0} {1}\n".format(output_prompt, data))
616 610
617 611 return out_data
618 612
619 613 def process_comment(self, data):
620 614 """Process data fPblock for COMMENT token."""
621 615 if not self.is_suppress:
622 616 return [data]
623 617
624 618 def save_image(self, image_file):
625 619 """
626 620 Saves the image file to disk.
627 621 """
628 622 self.ensure_pyplot()
629 623 command = 'plt.gcf().savefig("%s")'%image_file
630 624 #print 'SAVEFIG', command # dbg
631 625 self.process_input_line('bookmark ipy_thisdir', store_history=False)
632 626 self.process_input_line('cd -b ipy_savedir', store_history=False)
633 627 self.process_input_line(command, store_history=False)
634 628 self.process_input_line('cd -b ipy_thisdir', store_history=False)
635 629 self.process_input_line('bookmark -d ipy_thisdir', store_history=False)
636 630 self.clear_cout()
637 631
638 632 def process_block(self, block):
639 633 """
640 634 process block from the block_parser and return a list of processed lines
641 635 """
642 636 ret = []
643 637 output = None
644 638 input_lines = None
645 639 lineno = self.IP.execution_count
646 640
647 641 input_prompt = self.promptin % lineno
648 642 output_prompt = self.promptout % lineno
649 643 image_file = None
650 644 image_directive = None
651 645
652 646 found_input = False
653 647 for token, data in block:
654 648 if token == COMMENT:
655 649 out_data = self.process_comment(data)
656 650 elif token == INPUT:
657 651 found_input = True
658 652 (out_data, input_lines, output, is_doctest,
659 653 decorator, image_file, image_directive) = \
660 654 self.process_input(data, input_prompt, lineno)
661 655 elif token == OUTPUT:
662 656 if not found_input:
663 657
664 658 TAB = ' ' * 4
665 659 linenumber = 0
666 660 source = 'Unavailable'
667 661 content = 'Unavailable'
668 662 if self.directive:
669 663 linenumber = self.directive.state.document.current_line
670 664 source = self.directive.state.document.current_source
671 665 content = self.directive.content
672 666 # Add tabs and join into a single string.
673 667 content = '\n'.join([TAB + line for line in content])
674 668
675 669 e = ('\n\nInvalid block: Block contains an output prompt '
676 670 'without an input prompt.\n\n'
677 671 'Document source: {0}\n\n'
678 672 'Content begins at line {1}: \n\n{2}\n\n'
679 673 'Problematic block within content: \n\n{TAB}{3}\n\n')
680 674 e = e.format(source, linenumber, content, block, TAB=TAB)
681 675
682 676 # Write, rather than include in exception, since Sphinx
683 677 # will truncate tracebacks.
684 678 sys.stdout.write(e)
685 679 raise RuntimeError('An invalid block was detected.')
686 680
687 681 out_data = \
688 682 self.process_output(data, output_prompt, input_lines,
689 683 output, is_doctest, decorator,
690 684 image_file)
691 685 if out_data:
692 686 # Then there was user submitted output in verbatim mode.
693 687 # We need to remove the last element of `ret` that was
694 688 # added in `process_input`, as it is '' and would introduce
695 689 # an undesirable newline.
696 690 assert(ret[-1] == '')
697 691 del ret[-1]
698 692
699 693 if out_data:
700 694 ret.extend(out_data)
701 695
702 696 # save the image files
703 697 if image_file is not None:
704 698 self.save_image(image_file)
705 699
706 700 return ret, image_directive
707 701
708 702 def ensure_pyplot(self):
709 703 """
710 704 Ensures that pyplot has been imported into the embedded IPython shell.
711 705
712 706 Also, makes sure to set the backend appropriately if not set already.
713 707
714 708 """
715 709 # We are here if the @figure pseudo decorator was used. Thus, it's
716 710 # possible that we could be here even if python_mplbackend were set to
717 711 # `None`. That's also strange and perhaps worthy of raising an
718 712 # exception, but for now, we just set the backend to 'agg'.
719 713
720 714 if not self._pyplot_imported:
721 715 if 'matplotlib.backends' not in sys.modules:
722 716 # Then ipython_matplotlib was set to None but there was a
723 717 # call to the @figure decorator (and ipython_execlines did
724 718 # not set a backend).
725 719 #raise Exception("No backend was set, but @figure was used!")
726 720 import matplotlib
727 721 matplotlib.use('agg')
728 722
729 723 # Always import pyplot into embedded shell.
730 724 self.process_input_line('import matplotlib.pyplot as plt',
731 725 store_history=False)
732 726 self._pyplot_imported = True
733 727
734 728 def process_pure_python(self, content):
735 729 """
736 730 content is a list of strings. it is unedited directive content
737 731
738 732 This runs it line by line in the InteractiveShell, prepends
739 733 prompts as needed capturing stderr and stdout, then returns
740 734 the content as a list as if it were ipython code
741 735 """
742 736 output = []
743 737 savefig = False # keep up with this to clear figure
744 738 multiline = False # to handle line continuation
745 739 multiline_start = None
746 740 fmtin = self.promptin
747 741
748 742 ct = 0
749 743
750 744 for lineno, line in enumerate(content):
751 745
752 746 line_stripped = line.strip()
753 747 if not len(line):
754 748 output.append(line)
755 749 continue
756 750
757 751 # handle decorators
758 752 if line_stripped.startswith('@'):
759 753 output.extend([line])
760 754 if 'savefig' in line:
761 755 savefig = True # and need to clear figure
762 756 continue
763 757
764 758 # handle comments
765 759 if line_stripped.startswith('#'):
766 760 output.extend([line])
767 761 continue
768 762
769 763 # deal with lines checking for multiline
770 764 continuation = u' %s:'% ''.join(['.']*(len(str(ct))+2))
771 765 if not multiline:
772 766 modified = u"%s %s" % (fmtin % ct, line_stripped)
773 767 output.append(modified)
774 768 ct += 1
775 769 try:
776 770 ast.parse(line_stripped)
777 771 output.append(u'')
778 772 except Exception: # on a multiline
779 773 multiline = True
780 774 multiline_start = lineno
781 775 else: # still on a multiline
782 776 modified = u'%s %s' % (continuation, line)
783 777 output.append(modified)
784 778
785 779 # if the next line is indented, it should be part of multiline
786 780 if len(content) > lineno + 1:
787 781 nextline = content[lineno + 1]
788 782 if len(nextline) - len(nextline.lstrip()) > 3:
789 783 continue
790 784 try:
791 785 mod = ast.parse(
792 786 '\n'.join(content[multiline_start:lineno+1]))
793 787 if isinstance(mod.body[0], ast.FunctionDef):
794 788 # check to see if we have the whole function
795 789 for element in mod.body[0].body:
796 790 if isinstance(element, ast.Return):
797 791 multiline = False
798 792 else:
799 793 output.append(u'')
800 794 multiline = False
801 795 except Exception:
802 796 pass
803 797
804 798 if savefig: # clear figure if plotted
805 799 self.ensure_pyplot()
806 800 self.process_input_line('plt.clf()', store_history=False)
807 801 self.clear_cout()
808 802 savefig = False
809 803
810 804 return output
811 805
812 806 def custom_doctest(self, decorator, input_lines, found, submitted):
813 807 """
814 808 Perform a specialized doctest.
815 809
816 810 """
817 811 from .custom_doctests import doctests
818 812
819 813 args = decorator.split()
820 814 doctest_type = args[1]
821 815 if doctest_type in doctests:
822 816 doctests[doctest_type](self, args, input_lines, found, submitted)
823 817 else:
824 818 e = "Invalid option to @doctest: {0}".format(doctest_type)
825 819 raise Exception(e)
826 820
827 821
828 822 class IPythonDirective(Directive):
829 823
830 824 has_content = True
831 825 required_arguments = 0
832 826 optional_arguments = 4 # python, suppress, verbatim, doctest
833 827 final_argumuent_whitespace = True
834 828 option_spec = { 'python': directives.unchanged,
835 829 'suppress' : directives.flag,
836 830 'verbatim' : directives.flag,
837 831 'doctest' : directives.flag,
838 832 'okexcept': directives.flag,
839 833 'okwarning': directives.flag
840 834 }
841 835
842 836 shell = None
843 837
844 838 seen_docs = set()
845 839
846 840 def get_config_options(self):
847 841 # contains sphinx configuration variables
848 842 config = self.state.document.settings.env.config
849 843
850 844 # get config variables to set figure output directory
851 845 outdir = self.state.document.settings.env.app.outdir
852 846 savefig_dir = config.ipython_savefig_dir
853 847 source_dir = os.path.dirname(self.state.document.current_source)
854 848 if savefig_dir is None:
855 849 savefig_dir = config.html_static_path or '_static'
856 850 if isinstance(savefig_dir, list):
857 851 savefig_dir = os.path.join(*savefig_dir)
858 852 savefig_dir = os.path.join(outdir, savefig_dir)
859 853
860 854 # get regex and prompt stuff
861 855 rgxin = config.ipython_rgxin
862 856 rgxout = config.ipython_rgxout
863 857 promptin = config.ipython_promptin
864 858 promptout = config.ipython_promptout
865 859 mplbackend = config.ipython_mplbackend
866 860 exec_lines = config.ipython_execlines
867 861 hold_count = config.ipython_holdcount
868 862
869 863 return (savefig_dir, source_dir, rgxin, rgxout,
870 864 promptin, promptout, mplbackend, exec_lines, hold_count)
871 865
872 866 def setup(self):
873 867 # Get configuration values.
874 868 (savefig_dir, source_dir, rgxin, rgxout, promptin, promptout,
875 869 mplbackend, exec_lines, hold_count) = self.get_config_options()
876 870
877 871 if self.shell is None:
878 872 # We will be here many times. However, when the
879 873 # EmbeddedSphinxShell is created, its interactive shell member
880 874 # is the same for each instance.
881 875
882 876 if mplbackend and 'matplotlib.backends' not in sys.modules:
883 877 import matplotlib
884 878 matplotlib.use(mplbackend)
885 879
886 880 # Must be called after (potentially) importing matplotlib and
887 881 # setting its backend since exec_lines might import pylab.
888 882 self.shell = EmbeddedSphinxShell(exec_lines)
889 883
890 884 # Store IPython directive to enable better error messages
891 885 self.shell.directive = self
892 886
893 887 # reset the execution count if we haven't processed this doc
894 888 #NOTE: this may be borked if there are multiple seen_doc tmp files
895 889 #check time stamp?
896 890 if not self.state.document.current_source in self.seen_docs:
897 891 self.shell.IP.history_manager.reset()
898 892 self.shell.IP.execution_count = 1
899 893 self.seen_docs.add(self.state.document.current_source)
900 894
901 895 # and attach to shell so we don't have to pass them around
902 896 self.shell.rgxin = rgxin
903 897 self.shell.rgxout = rgxout
904 898 self.shell.promptin = promptin
905 899 self.shell.promptout = promptout
906 900 self.shell.savefig_dir = savefig_dir
907 901 self.shell.source_dir = source_dir
908 902 self.shell.hold_count = hold_count
909 903
910 904 # setup bookmark for saving figures directory
911 905 self.shell.process_input_line('bookmark ipy_savedir %s'%savefig_dir,
912 906 store_history=False)
913 907 self.shell.clear_cout()
914 908
915 909 return rgxin, rgxout, promptin, promptout
916 910
917 911 def teardown(self):
918 912 # delete last bookmark
919 913 self.shell.process_input_line('bookmark -d ipy_savedir',
920 914 store_history=False)
921 915 self.shell.clear_cout()
922 916
923 917 def run(self):
924 918 debug = False
925 919
926 920 #TODO, any reason block_parser can't be a method of embeddable shell
927 921 # then we wouldn't have to carry these around
928 922 rgxin, rgxout, promptin, promptout = self.setup()
929 923
930 924 options = self.options
931 925 self.shell.is_suppress = 'suppress' in options
932 926 self.shell.is_doctest = 'doctest' in options
933 927 self.shell.is_verbatim = 'verbatim' in options
934 928 self.shell.is_okexcept = 'okexcept' in options
935 929 self.shell.is_okwarning = 'okwarning' in options
936 930
937 931 # handle pure python code
938 932 if 'python' in self.arguments:
939 933 content = self.content
940 934 self.content = self.shell.process_pure_python(content)
941 935
942 936 # parts consists of all text within the ipython-block.
943 937 # Each part is an input/output block.
944 938 parts = '\n'.join(self.content).split('\n\n')
945 939
946 940 lines = ['.. code-block:: ipython', '']
947 941 figures = []
948 942
949 943 for part in parts:
950 944 block = block_parser(part, rgxin, rgxout, promptin, promptout)
951 945 if len(block):
952 946 rows, figure = self.shell.process_block(block)
953 947 for row in rows:
954 948 lines.extend([' {0}'.format(line)
955 949 for line in row.split('\n')])
956 950
957 951 if figure is not None:
958 952 figures.append(figure)
959 953
960 954 for figure in figures:
961 955 lines.append('')
962 956 lines.extend(figure.split('\n'))
963 957 lines.append('')
964 958
965 959 if len(lines) > 2:
966 960 if debug:
967 961 print('\n'.join(lines))
968 962 else:
969 963 # This has to do with input, not output. But if we comment
970 964 # these lines out, then no IPython code will appear in the
971 965 # final output.
972 966 self.state_machine.insert_input(
973 967 lines, self.state_machine.input_lines.source(0))
974 968
975 969 # cleanup
976 970 self.teardown()
977 971
978 972 return []
979 973
980 974 # Enable as a proper Sphinx directive
981 975 def setup(app):
982 976 setup.app = app
983 977
984 978 app.add_directive('ipython', IPythonDirective)
985 979 app.add_config_value('ipython_savefig_dir', None, 'env')
986 980 app.add_config_value('ipython_rgxin',
987 981 re.compile('In \[(\d+)\]:\s?(.*)\s*'), 'env')
988 982 app.add_config_value('ipython_rgxout',
989 983 re.compile('Out\[(\d+)\]:\s?(.*)\s*'), 'env')
990 984 app.add_config_value('ipython_promptin', 'In [%d]:', 'env')
991 985 app.add_config_value('ipython_promptout', 'Out[%d]:', 'env')
992 986
993 987 # We could just let matplotlib pick whatever is specified as the default
994 988 # backend in the matplotlibrc file, but this would cause issues if the
995 989 # backend didn't work in headless environments. For this reason, 'agg'
996 990 # is a good default backend choice.
997 991 app.add_config_value('ipython_mplbackend', 'agg', 'env')
998 992
999 993 # If the user sets this config value to `None`, then EmbeddedSphinxShell's
1000 994 # __init__ method will treat it as [].
1001 995 execlines = ['import numpy as np', 'import matplotlib.pyplot as plt']
1002 996 app.add_config_value('ipython_execlines', execlines, 'env')
1003 997
1004 998 app.add_config_value('ipython_holdcount', True, 'env')
1005 999
1006 1000 metadata = {'parallel_read_safe': True, 'parallel_write_safe': True}
1007 1001 return metadata
1008 1002
1009 1003 # Simple smoke test, needs to be converted to a proper automatic test.
1010 1004 def test():
1011 1005
1012 1006 examples = [
1013 1007 r"""
1014 1008 In [9]: pwd
1015 1009 Out[9]: '/home/jdhunter/py4science/book'
1016 1010
1017 1011 In [10]: cd bookdata/
1018 1012 /home/jdhunter/py4science/book/bookdata
1019 1013
1020 1014 In [2]: from pylab import *
1021 1015
1022 1016 In [2]: ion()
1023 1017
1024 1018 In [3]: im = imread('stinkbug.png')
1025 1019
1026 1020 @savefig mystinkbug.png width=4in
1027 1021 In [4]: imshow(im)
1028 1022 Out[4]: <matplotlib.image.AxesImage object at 0x39ea850>
1029 1023
1030 1024 """,
1031 1025 r"""
1032 1026
1033 1027 In [1]: x = 'hello world'
1034 1028
1035 1029 # string methods can be
1036 1030 # used to alter the string
1037 1031 @doctest
1038 1032 In [2]: x.upper()
1039 1033 Out[2]: 'HELLO WORLD'
1040 1034
1041 1035 @verbatim
1042 1036 In [3]: x.st<TAB>
1043 1037 x.startswith x.strip
1044 1038 """,
1045 1039 r"""
1046 1040
1047 1041 In [130]: url = 'http://ichart.finance.yahoo.com/table.csv?s=CROX\
1048 1042 .....: &d=9&e=22&f=2009&g=d&a=1&br=8&c=2006&ignore=.csv'
1049 1043
1050 1044 In [131]: print url.split('&')
1051 1045 ['http://ichart.finance.yahoo.com/table.csv?s=CROX', 'd=9', 'e=22', 'f=2009', 'g=d', 'a=1', 'b=8', 'c=2006', 'ignore=.csv']
1052 1046
1053 1047 In [60]: import urllib
1054 1048
1055 1049 """,
1056 1050 r"""\
1057 1051
1058 1052 In [133]: import numpy.random
1059 1053
1060 1054 @suppress
1061 1055 In [134]: numpy.random.seed(2358)
1062 1056
1063 1057 @doctest
1064 1058 In [135]: numpy.random.rand(10,2)
1065 1059 Out[135]:
1066 1060 array([[ 0.64524308, 0.59943846],
1067 1061 [ 0.47102322, 0.8715456 ],
1068 1062 [ 0.29370834, 0.74776844],
1069 1063 [ 0.99539577, 0.1313423 ],
1070 1064 [ 0.16250302, 0.21103583],
1071 1065 [ 0.81626524, 0.1312433 ],
1072 1066 [ 0.67338089, 0.72302393],
1073 1067 [ 0.7566368 , 0.07033696],
1074 1068 [ 0.22591016, 0.77731835],
1075 1069 [ 0.0072729 , 0.34273127]])
1076 1070
1077 1071 """,
1078 1072
1079 1073 r"""
1080 1074 In [106]: print x
1081 1075 jdh
1082 1076
1083 1077 In [109]: for i in range(10):
1084 1078 .....: print i
1085 1079 .....:
1086 1080 .....:
1087 1081 0
1088 1082 1
1089 1083 2
1090 1084 3
1091 1085 4
1092 1086 5
1093 1087 6
1094 1088 7
1095 1089 8
1096 1090 9
1097 1091 """,
1098 1092
1099 1093 r"""
1100 1094
1101 1095 In [144]: from pylab import *
1102 1096
1103 1097 In [145]: ion()
1104 1098
1105 1099 # use a semicolon to suppress the output
1106 1100 @savefig test_hist.png width=4in
1107 1101 In [151]: hist(np.random.randn(10000), 100);
1108 1102
1109 1103
1110 1104 @savefig test_plot.png width=4in
1111 1105 In [151]: plot(np.random.randn(10000), 'o');
1112 1106 """,
1113 1107
1114 1108 r"""
1115 1109 # use a semicolon to suppress the output
1116 1110 In [151]: plt.clf()
1117 1111
1118 1112 @savefig plot_simple.png width=4in
1119 1113 In [151]: plot([1,2,3])
1120 1114
1121 1115 @savefig hist_simple.png width=4in
1122 1116 In [151]: hist(np.random.randn(10000), 100);
1123 1117
1124 1118 """,
1125 1119 r"""
1126 1120 # update the current fig
1127 1121 In [151]: ylabel('number')
1128 1122
1129 1123 In [152]: title('normal distribution')
1130 1124
1131 1125
1132 1126 @savefig hist_with_text.png
1133 1127 In [153]: grid(True)
1134 1128
1135 1129 @doctest float
1136 1130 In [154]: 0.1 + 0.2
1137 1131 Out[154]: 0.3
1138 1132
1139 1133 @doctest float
1140 1134 In [155]: np.arange(16).reshape(4,4)
1141 1135 Out[155]:
1142 1136 array([[ 0, 1, 2, 3],
1143 1137 [ 4, 5, 6, 7],
1144 1138 [ 8, 9, 10, 11],
1145 1139 [12, 13, 14, 15]])
1146 1140
1147 1141 In [1]: x = np.arange(16, dtype=float).reshape(4,4)
1148 1142
1149 1143 In [2]: x[0,0] = np.inf
1150 1144
1151 1145 In [3]: x[0,1] = np.nan
1152 1146
1153 1147 @doctest float
1154 1148 In [4]: x
1155 1149 Out[4]:
1156 1150 array([[ inf, nan, 2., 3.],
1157 1151 [ 4., 5., 6., 7.],
1158 1152 [ 8., 9., 10., 11.],
1159 1153 [ 12., 13., 14., 15.]])
1160 1154
1161 1155
1162 1156 """,
1163 1157 ]
1164 1158 # skip local-file depending first example:
1165 1159 examples = examples[1:]
1166 1160
1167 1161 #ipython_directive.DEBUG = True # dbg
1168 1162 #options = dict(suppress=True) # dbg
1169 1163 options = dict()
1170 1164 for example in examples:
1171 1165 content = example.split('\n')
1172 1166 IPythonDirective('debug', arguments=None, options=options,
1173 1167 content=content, lineno=0,
1174 1168 content_offset=None, block_text=None,
1175 1169 state=None, state_machine=None,
1176 1170 )
1177 1171
1178 1172 # Run test suite as a script
1179 1173 if __name__=='__main__':
1180 1174 if not os.path.isdir('_static'):
1181 1175 os.mkdir('_static')
1182 1176 test()
1183 1177 print('All OK? Check figures in _static/')
General Comments 0
You need to be logged in to leave comments. Login now