##// END OF EJS Templates
add additional clarification in ip_d re prompt stripping
y-p -
Show More
@@ -1,1050 +1,1053 b''
1 1 # -*- coding: utf-8 -*-
2 2 """
3 3 Sphinx directive to support embedded IPython code.
4 4
5 5 This directive allows pasting of entire interactive IPython sessions, prompts
6 6 and all, and their code will actually get re-executed at doc build time, with
7 7 all prompts renumbered sequentially. It also allows you to input code as a pure
8 8 python input by giving the argument python to the directive. The output looks
9 9 like an interactive ipython section.
10 10
11 11 To enable this directive, simply list it in your Sphinx ``conf.py`` file
12 12 (making sure the directory where you placed it is visible to sphinx, as is
13 13 needed for all Sphinx directives). For example, to enable syntax highlighting
14 14 and the IPython directive::
15 15
16 16 extensions = ['IPython.sphinxext.ipython_console_highlighting',
17 17 'IPython.sphinxext.ipython_directive']
18 18
19 19 The IPython directive outputs code-blocks with the language 'ipython'. So
20 20 if you do not have the syntax highlighting extension enabled as well, then
21 21 all rendered code-blocks will be uncolored. By default this directive assumes
22 22 that your prompts are unchanged IPython ones, but this can be customized.
23 23 The configurable options that can be placed in conf.py are:
24 24
25 25 ipython_savefig_dir:
26 26 The directory in which to save the figures. This is relative to the
27 27 Sphinx source directory. The default is `html_static_path`.
28 28 ipython_rgxin:
29 29 The compiled regular expression to denote the start of IPython input
30 30 lines. The default is re.compile('In \[(\d+)\]:\s?(.*)\s*'). You
31 31 shouldn't need to change this.
32 32 ipython_rgxout:
33 33 The compiled regular expression to denote the start of IPython output
34 34 lines. The default is re.compile('Out\[(\d+)\]:\s?(.*)\s*'). You
35 35 shouldn't need to change this.
36 36 ipython_promptin:
37 37 The string to represent the IPython input prompt in the generated ReST.
38 38 The default is 'In [%d]:'. This expects that the line numbers are used
39 39 in the prompt.
40 40 ipython_promptout:
41 41 The string to represent the IPython prompt in the generated ReST. The
42 42 default is 'Out [%d]:'. This expects that the line numbers are used
43 43 in the prompt.
44 44 ipython_mplbackend:
45 45 The string which specifies if the embedded Sphinx shell should import
46 46 Matplotlib and set the backend. The value specifies a backend that is
47 47 passed to `matplotlib.use()` before any lines in `ipython_execlines` are
48 48 executed. If not specified in conf.py, then the default value of 'agg' is
49 49 used. To use the IPython directive without matplotlib as a dependency, set
50 50 the value to `None`. It may end up that matplotlib is still imported
51 51 if the user specifies so in `ipython_execlines` or makes use of the
52 52 @savefig pseudo decorator.
53 53 ipython_execlines:
54 54 A list of strings to be exec'd in the embedded Sphinx shell. Typical
55 55 usage is to make certain packages always available. Set this to an empty
56 56 list if you wish to have no imports always available. If specified in
57 57 conf.py as `None`, then it has the effect of making no imports available.
58 58 If omitted from conf.py altogether, then the default value of
59 59 ['import numpy as np', 'import matplotlib.pyplot as plt'] is used.
60 60 ipython_holdcount
61 61 When the @suppress pseudo-decorator is used, the execution count can be
62 62 incremented or not. The default behavior is to hold the execution count,
63 63 corresponding to a value of `True`. Set this to `False` to increment
64 64 the execution count after each suppressed command.
65 65
66 66 As an example, to use the IPython directive when `matplotlib` is not available,
67 67 one sets the backend to `None`::
68 68
69 69 ipython_mplbackend = None
70 70
71 71 An example usage of the directive is:
72 72
73 73 .. code-block:: rst
74 74
75 75 .. ipython::
76 76
77 77 In [1]: x = 1
78 78
79 79 In [2]: y = x**2
80 80
81 81 In [3]: print(y)
82 82
83 83 See http://matplotlib.org/sampledoc/ipython_directive.html for additional
84 84 documentation.
85 85
86 86 ToDo
87 87 ----
88 88
89 89 - Turn the ad-hoc test() function into a real test suite.
90 90 - Break up ipython-specific functionality from matplotlib stuff into better
91 91 separated code.
92 92
93 93 Authors
94 94 -------
95 95
96 96 - John D Hunter: orignal author.
97 97 - Fernando Perez: refactoring, documentation, cleanups, port to 0.11.
98 98 - VΓ‘clavΕ milauer <eudoxos-AT-arcig.cz>: Prompt generalizations.
99 99 - Skipper Seabold, refactoring, cleanups, pure python addition
100 100 """
101 101 from __future__ import print_function
102 102
103 103 #-----------------------------------------------------------------------------
104 104 # Imports
105 105 #-----------------------------------------------------------------------------
106 106
107 107 # Stdlib
108 108 import os
109 109 import re
110 110 import sys
111 111 import tempfile
112 112 import ast
113 113 import warnings
114 114
115 115 # To keep compatibility with various python versions
116 116 try:
117 117 from hashlib import md5
118 118 except ImportError:
119 119 from md5 import md5
120 120
121 121 # Third-party
122 122 import sphinx
123 123 from docutils.parsers.rst import directives
124 124 from docutils import nodes
125 125 from sphinx.util.compat import Directive
126 126
127 127 # Our own
128 128 from IPython import Config, InteractiveShell
129 129 from IPython.core.profiledir import ProfileDir
130 130 from IPython.utils import io
131 131 from IPython.utils.py3compat import PY3
132 132
133 133 if PY3:
134 134 from io import StringIO
135 135 else:
136 136 from StringIO import StringIO
137 137
138 138 #-----------------------------------------------------------------------------
139 139 # Globals
140 140 #-----------------------------------------------------------------------------
141 141 # for tokenizing blocks
142 142 COMMENT, INPUT, OUTPUT = range(3)
143 143
144 144 #-----------------------------------------------------------------------------
145 145 # Functions and class declarations
146 146 #-----------------------------------------------------------------------------
147 147
148 148 def block_parser(part, rgxin, rgxout, fmtin, fmtout):
149 149 """
150 150 part is a string of ipython text, comprised of at most one
151 151 input, one ouput, comments, and blank lines. The block parser
152 152 parses the text into a list of::
153 153
154 154 blocks = [ (TOKEN0, data0), (TOKEN1, data1), ...]
155 155
156 156 where TOKEN is one of [COMMENT | INPUT | OUTPUT ] and
157 157 data is, depending on the type of token::
158 158
159 159 COMMENT : the comment string
160 160
161 161 INPUT: the (DECORATOR, INPUT_LINE, REST) where
162 162 DECORATOR: the input decorator (or None)
163 163 INPUT_LINE: the input as string (possibly multi-line)
164 164 REST : any stdout generated by the input line (not OUTPUT)
165 165
166 166 OUTPUT: the output string, possibly multi-line
167 167
168 168 """
169 169 block = []
170 170 lines = part.split('\n')
171 171 N = len(lines)
172 172 i = 0
173 173 decorator = None
174 174 while 1:
175 175
176 176 if i==N:
177 177 # nothing left to parse -- the last line
178 178 break
179 179
180 180 line = lines[i]
181 181 i += 1
182 182 line_stripped = line.strip()
183 183 if line_stripped.startswith('#'):
184 184 block.append((COMMENT, line))
185 185 continue
186 186
187 187 if line_stripped.startswith('@'):
188 188 # we're assuming at most one decorator -- may need to
189 189 # rethink
190 190 decorator = line_stripped
191 191 continue
192 192
193 193 # does this look like an input line?
194 194 matchin = rgxin.match(line)
195 195 if matchin:
196 196 lineno, inputline = int(matchin.group(1)), matchin.group(2)
197 197
198 198 # the ....: continuation string
199 199 continuation = ' %s:'%''.join(['.']*(len(str(lineno))+2))
200 200 Nc = len(continuation)
201 201 # input lines can continue on for more than one line, if
202 202 # we have a '\' line continuation char or a function call
203 203 # echo line 'print'. The input line can only be
204 204 # terminated by the end of the block or an output line, so
205 205 # we parse out the rest of the input line if it is
206 206 # multiline as well as any echo text
207 207
208 208 rest = []
209 209 while i<N:
210 210
211 211 # look ahead; if the next line is blank, or a comment, or
212 212 # an output line, we're done
213 213
214 214 nextline = lines[i]
215 215 matchout = rgxout.match(nextline)
216 216 #print "nextline=%s, continuation=%s, starts=%s"%(nextline, continuation, nextline.startswith(continuation))
217 217 if matchout or nextline.startswith('#'):
218 218 break
219 219 elif nextline.startswith(continuation):
220 # ipython_rgxout says that the space after the colon is optional
221 # If the space is there we must consume it, otherwise code
222 # employing the cython_magic extension fails to work.
220 # The default ipython_rgx* treat the space following the colon as optional.
221 # However, If the space is there we must consume it or code
222 # employing the cython_magic extension will fail to execute.
223 #
224 # This works with the default ipython_rgx* patterns,
225 # If you modify them, YMMV.
223 226 nextline = nextline[Nc:]
224 227 if nextline and nextline[0] == ' ':
225 228 nextline = nextline[1:]
226 229
227 230 inputline += '\n' + nextline
228 231 else:
229 232 rest.append(nextline)
230 233 i+= 1
231 234
232 235 block.append((INPUT, (decorator, inputline, '\n'.join(rest))))
233 236 continue
234 237
235 238 # if it looks like an output line grab all the text to the end
236 239 # of the block
237 240 matchout = rgxout.match(line)
238 241 if matchout:
239 242 lineno, output = int(matchout.group(1)), matchout.group(2)
240 243 if i<N-1:
241 244 output = '\n'.join([output] + lines[i:])
242 245
243 246 block.append((OUTPUT, output))
244 247 break
245 248
246 249 return block
247 250
248 251
249 252 class EmbeddedSphinxShell(object):
250 253 """An embedded IPython instance to run inside Sphinx"""
251 254
252 255 def __init__(self, exec_lines=None):
253 256
254 257 self.cout = StringIO()
255 258
256 259 if exec_lines is None:
257 260 exec_lines = []
258 261
259 262 # Create config object for IPython
260 263 config = Config()
261 264 config.InteractiveShell.autocall = False
262 265 config.InteractiveShell.autoindent = False
263 266 config.InteractiveShell.colors = 'NoColor'
264 267
265 268 # create a profile so instance history isn't saved
266 269 tmp_profile_dir = tempfile.mkdtemp(prefix='profile_')
267 270 profname = 'auto_profile_sphinx_build'
268 271 pdir = os.path.join(tmp_profile_dir,profname)
269 272 profile = ProfileDir.create_profile_dir(pdir)
270 273
271 274 # Create and initialize global ipython, but don't start its mainloop.
272 275 # This will persist across different EmbededSphinxShell instances.
273 276 IP = InteractiveShell.instance(config=config, profile_dir=profile)
274 277
275 278 # io.stdout redirect must be done after instantiating InteractiveShell
276 279 io.stdout = self.cout
277 280 io.stderr = self.cout
278 281
279 282 # For debugging, so we can see normal output, use this:
280 283 #from IPython.utils.io import Tee
281 284 #io.stdout = Tee(self.cout, channel='stdout') # dbg
282 285 #io.stderr = Tee(self.cout, channel='stderr') # dbg
283 286
284 287 # Store a few parts of IPython we'll need.
285 288 self.IP = IP
286 289 self.user_ns = self.IP.user_ns
287 290 self.user_global_ns = self.IP.user_global_ns
288 291
289 292 self.input = ''
290 293 self.output = ''
291 294
292 295 self.is_verbatim = False
293 296 self.is_doctest = False
294 297 self.is_suppress = False
295 298
296 299 # Optionally, provide more detailed information to shell.
297 300 # this is assigned by the SetUp method of IPythonDirective
298 301 # to point at itself.
299 302 #
300 303 # So, you can access handy things at self.directive.state
301 304 self.directive = None
302 305
303 306 # on the first call to the savefig decorator, we'll import
304 307 # pyplot as plt so we can make a call to the plt.gcf().savefig
305 308 self._pyplot_imported = False
306 309
307 310 # Prepopulate the namespace.
308 311 for line in exec_lines:
309 312 self.process_input_line(line, store_history=False)
310 313
311 314 def clear_cout(self):
312 315 self.cout.seek(0)
313 316 self.cout.truncate(0)
314 317
315 318 def process_input_line(self, line, store_history=True):
316 319 """process the input, capturing stdout"""
317 320
318 321 stdout = sys.stdout
319 322 splitter = self.IP.input_splitter
320 323 try:
321 324 sys.stdout = self.cout
322 325 splitter.push(line)
323 326 more = splitter.push_accepts_more()
324 327 if not more:
325 328 source_raw = splitter.source_raw_reset()[1]
326 329 self.IP.run_cell(source_raw, store_history=store_history)
327 330 finally:
328 331 sys.stdout = stdout
329 332
330 333 def process_image(self, decorator):
331 334 """
332 335 # build out an image directive like
333 336 # .. image:: somefile.png
334 337 # :width 4in
335 338 #
336 339 # from an input like
337 340 # savefig somefile.png width=4in
338 341 """
339 342 savefig_dir = self.savefig_dir
340 343 source_dir = self.source_dir
341 344 saveargs = decorator.split(' ')
342 345 filename = saveargs[1]
343 346 # insert relative path to image file in source
344 347 outfile = os.path.relpath(os.path.join(savefig_dir,filename),
345 348 source_dir)
346 349
347 350 imagerows = ['.. image:: %s'%outfile]
348 351
349 352 for kwarg in saveargs[2:]:
350 353 arg, val = kwarg.split('=')
351 354 arg = arg.strip()
352 355 val = val.strip()
353 356 imagerows.append(' :%s: %s'%(arg, val))
354 357
355 358 image_file = os.path.basename(outfile) # only return file name
356 359 image_directive = '\n'.join(imagerows)
357 360 return image_file, image_directive
358 361
359 362 # Callbacks for each type of token
360 363 def process_input(self, data, input_prompt, lineno):
361 364 """
362 365 Process data block for INPUT token.
363 366
364 367 """
365 368 decorator, input, rest = data
366 369 image_file = None
367 370 image_directive = None
368 371
369 372 is_verbatim = decorator=='@verbatim' or self.is_verbatim
370 373 is_doctest = (decorator is not None and \
371 374 decorator.startswith('@doctest')) or self.is_doctest
372 375 is_suppress = decorator=='@suppress' or self.is_suppress
373 376 is_okexcept = decorator=='@okexcept' or self.is_okexcept
374 377 is_okwarning = decorator=='@okwarning' or self.is_okwarning
375 378 is_savefig = decorator is not None and \
376 379 decorator.startswith('@savefig')
377 380
378 381 input_lines = input.split('\n')
379 382 if len(input_lines) > 1:
380 383 if input_lines[-1] != "":
381 384 input_lines.append('') # make sure there's a blank line
382 385 # so splitter buffer gets reset
383 386
384 387 continuation = ' %s:'%''.join(['.']*(len(str(lineno))+2))
385 388
386 389 if is_savefig:
387 390 image_file, image_directive = self.process_image(decorator)
388 391
389 392 ret = []
390 393 is_semicolon = False
391 394
392 395 # Hold the execution count, if requested to do so.
393 396 if is_suppress and self.hold_count:
394 397 store_history = False
395 398 else:
396 399 store_history = True
397 400
398 401 # Note: catch_warnings is not thread safe
399 402 with warnings.catch_warnings(record=True) as ws:
400 403 for i, line in enumerate(input_lines):
401 404 if line.endswith(';'):
402 405 is_semicolon = True
403 406
404 407 if i == 0:
405 408 # process the first input line
406 409 if is_verbatim:
407 410 self.process_input_line('')
408 411 self.IP.execution_count += 1 # increment it anyway
409 412 else:
410 413 # only submit the line in non-verbatim mode
411 414 self.process_input_line(line, store_history=store_history)
412 415 formatted_line = '%s %s'%(input_prompt, line)
413 416 else:
414 417 # process a continuation line
415 418 if not is_verbatim:
416 419 self.process_input_line(line, store_history=store_history)
417 420
418 421 formatted_line = '%s %s'%(continuation, line)
419 422
420 423 if not is_suppress:
421 424 ret.append(formatted_line)
422 425
423 426 if not is_suppress and len(rest.strip()) and is_verbatim:
424 427 # the "rest" is the standard output of the
425 428 # input, which needs to be added in
426 429 # verbatim mode
427 430 ret.append(rest)
428 431
429 432 self.cout.seek(0)
430 433 output = self.cout.read()
431 434 if not is_suppress and not is_semicolon:
432 435 ret.append(output)
433 436 elif is_semicolon: # get spacing right
434 437 ret.append('')
435 438
436 439 # context information
437 440 filename = "Unknown"
438 441 lineno = 0
439 442 if self.directive.state:
440 443 filename = self.directive.state.document.current_source
441 444 lineno = self.directive.state.document.current_line
442 445
443 446 # output any exceptions raised during execution to stdout
444 447 # unless :okexcept: has been specified.
445 448 if not is_okexcept and "Traceback" in output:
446 449 s = "\nException in %s at block ending on line %s\n" % (filename, lineno)
447 450 s += "Specify :okexcept: as an option in the ipython:: block to suppress this message\n"
448 451 sys.stdout.write('\n\n>>>' + ('-' * 73))
449 452 sys.stdout.write(s)
450 453 sys.stdout.write(output)
451 454 sys.stdout.write('<<<' + ('-' * 73) + '\n\n')
452 455
453 456 # output any warning raised during execution to stdout
454 457 # unless :okwarning: has been specified.
455 458 if not is_okwarning:
456 459 for w in ws:
457 460 s = "\nWarning in %s at block ending on line %s\n" % (filename, lineno)
458 461 s += "Specify :okwarning: as an option in the ipython:: block to suppress this message\n"
459 462 sys.stdout.write('\n\n>>>' + ('-' * 73))
460 463 sys.stdout.write(s)
461 464 sys.stdout.write(('-' * 76) + '\n')
462 465 s=warnings.formatwarning(w.message, w.category,
463 466 w.filename, w.lineno, w.line)
464 467 sys.stdout.write(s)
465 468 sys.stdout.write('<<<' + ('-' * 73) + '\n')
466 469
467 470 self.cout.truncate(0)
468 471 return (ret, input_lines, output, is_doctest, decorator, image_file,
469 472 image_directive)
470 473
471 474
472 475 def process_output(self, data, output_prompt,
473 476 input_lines, output, is_doctest, decorator, image_file):
474 477 """
475 478 Process data block for OUTPUT token.
476 479
477 480 """
478 481 TAB = ' ' * 4
479 482
480 483 if is_doctest and output is not None:
481 484
482 485 found = output
483 486 found = found.strip()
484 487 submitted = data.strip()
485 488
486 489 if self.directive is None:
487 490 source = 'Unavailable'
488 491 content = 'Unavailable'
489 492 else:
490 493 source = self.directive.state.document.current_source
491 494 content = self.directive.content
492 495 # Add tabs and join into a single string.
493 496 content = '\n'.join([TAB + line for line in content])
494 497
495 498 # Make sure the output contains the output prompt.
496 499 ind = found.find(output_prompt)
497 500 if ind < 0:
498 501 e = ('output does not contain output prompt\n\n'
499 502 'Document source: {0}\n\n'
500 503 'Raw content: \n{1}\n\n'
501 504 'Input line(s):\n{TAB}{2}\n\n'
502 505 'Output line(s):\n{TAB}{3}\n\n')
503 506 e = e.format(source, content, '\n'.join(input_lines),
504 507 repr(found), TAB=TAB)
505 508 raise RuntimeError(e)
506 509 found = found[len(output_prompt):].strip()
507 510
508 511 # Handle the actual doctest comparison.
509 512 if decorator.strip() == '@doctest':
510 513 # Standard doctest
511 514 if found != submitted:
512 515 e = ('doctest failure\n\n'
513 516 'Document source: {0}\n\n'
514 517 'Raw content: \n{1}\n\n'
515 518 'On input line(s):\n{TAB}{2}\n\n'
516 519 'we found output:\n{TAB}{3}\n\n'
517 520 'instead of the expected:\n{TAB}{4}\n\n')
518 521 e = e.format(source, content, '\n'.join(input_lines),
519 522 repr(found), repr(submitted), TAB=TAB)
520 523 raise RuntimeError(e)
521 524 else:
522 525 self.custom_doctest(decorator, input_lines, found, submitted)
523 526
524 527 def process_comment(self, data):
525 528 """Process data fPblock for COMMENT token."""
526 529 if not self.is_suppress:
527 530 return [data]
528 531
529 532 def save_image(self, image_file):
530 533 """
531 534 Saves the image file to disk.
532 535 """
533 536 self.ensure_pyplot()
534 537 command = 'plt.gcf().savefig("%s")'%image_file
535 538 #print 'SAVEFIG', command # dbg
536 539 self.process_input_line('bookmark ipy_thisdir', store_history=False)
537 540 self.process_input_line('cd -b ipy_savedir', store_history=False)
538 541 self.process_input_line(command, store_history=False)
539 542 self.process_input_line('cd -b ipy_thisdir', store_history=False)
540 543 self.process_input_line('bookmark -d ipy_thisdir', store_history=False)
541 544 self.clear_cout()
542 545
543 546 def process_block(self, block):
544 547 """
545 548 process block from the block_parser and return a list of processed lines
546 549 """
547 550 ret = []
548 551 output = None
549 552 input_lines = None
550 553 lineno = self.IP.execution_count
551 554
552 555 input_prompt = self.promptin % lineno
553 556 output_prompt = self.promptout % lineno
554 557 image_file = None
555 558 image_directive = None
556 559
557 560 for token, data in block:
558 561 if token == COMMENT:
559 562 out_data = self.process_comment(data)
560 563 elif token == INPUT:
561 564 (out_data, input_lines, output, is_doctest, decorator,
562 565 image_file, image_directive) = \
563 566 self.process_input(data, input_prompt, lineno)
564 567 elif token == OUTPUT:
565 568 out_data = \
566 569 self.process_output(data, output_prompt,
567 570 input_lines, output, is_doctest,
568 571 decorator, image_file)
569 572 if out_data:
570 573 ret.extend(out_data)
571 574
572 575 # save the image files
573 576 if image_file is not None:
574 577 self.save_image(image_file)
575 578
576 579 return ret, image_directive
577 580
578 581 def ensure_pyplot(self):
579 582 """
580 583 Ensures that pyplot has been imported into the embedded IPython shell.
581 584
582 585 Also, makes sure to set the backend appropriately if not set already.
583 586
584 587 """
585 588 # We are here if the @figure pseudo decorator was used. Thus, it's
586 589 # possible that we could be here even if python_mplbackend were set to
587 590 # `None`. That's also strange and perhaps worthy of raising an
588 591 # exception, but for now, we just set the backend to 'agg'.
589 592
590 593 if not self._pyplot_imported:
591 594 if 'matplotlib.backends' not in sys.modules:
592 595 # Then ipython_matplotlib was set to None but there was a
593 596 # call to the @figure decorator (and ipython_execlines did
594 597 # not set a backend).
595 598 #raise Exception("No backend was set, but @figure was used!")
596 599 import matplotlib
597 600 matplotlib.use('agg')
598 601
599 602 # Always import pyplot into embedded shell.
600 603 self.process_input_line('import matplotlib.pyplot as plt',
601 604 store_history=False)
602 605 self._pyplot_imported = True
603 606
604 607 def process_pure_python(self, content):
605 608 """
606 609 content is a list of strings. it is unedited directive content
607 610
608 611 This runs it line by line in the InteractiveShell, prepends
609 612 prompts as needed capturing stderr and stdout, then returns
610 613 the content as a list as if it were ipython code
611 614 """
612 615 output = []
613 616 savefig = False # keep up with this to clear figure
614 617 multiline = False # to handle line continuation
615 618 multiline_start = None
616 619 fmtin = self.promptin
617 620
618 621 ct = 0
619 622
620 623 for lineno, line in enumerate(content):
621 624
622 625 line_stripped = line.strip()
623 626 if not len(line):
624 627 output.append(line)
625 628 continue
626 629
627 630 # handle decorators
628 631 if line_stripped.startswith('@'):
629 632 output.extend([line])
630 633 if 'savefig' in line:
631 634 savefig = True # and need to clear figure
632 635 continue
633 636
634 637 # handle comments
635 638 if line_stripped.startswith('#'):
636 639 output.extend([line])
637 640 continue
638 641
639 642 # deal with lines checking for multiline
640 643 continuation = u' %s:'% ''.join(['.']*(len(str(ct))+2))
641 644 if not multiline:
642 645 modified = u"%s %s" % (fmtin % ct, line_stripped)
643 646 output.append(modified)
644 647 ct += 1
645 648 try:
646 649 ast.parse(line_stripped)
647 650 output.append(u'')
648 651 except Exception: # on a multiline
649 652 multiline = True
650 653 multiline_start = lineno
651 654 else: # still on a multiline
652 655 modified = u'%s %s' % (continuation, line)
653 656 output.append(modified)
654 657
655 658 # if the next line is indented, it should be part of multiline
656 659 if len(content) > lineno + 1:
657 660 nextline = content[lineno + 1]
658 661 if len(nextline) - len(nextline.lstrip()) > 3:
659 662 continue
660 663 try:
661 664 mod = ast.parse(
662 665 '\n'.join(content[multiline_start:lineno+1]))
663 666 if isinstance(mod.body[0], ast.FunctionDef):
664 667 # check to see if we have the whole function
665 668 for element in mod.body[0].body:
666 669 if isinstance(element, ast.Return):
667 670 multiline = False
668 671 else:
669 672 output.append(u'')
670 673 multiline = False
671 674 except Exception:
672 675 pass
673 676
674 677 if savefig: # clear figure if plotted
675 678 self.ensure_pyplot()
676 679 self.process_input_line('plt.clf()', store_history=False)
677 680 self.clear_cout()
678 681 savefig = False
679 682
680 683 return output
681 684
682 685 def custom_doctest(self, decorator, input_lines, found, submitted):
683 686 """
684 687 Perform a specialized doctest.
685 688
686 689 """
687 690 from .custom_doctests import doctests
688 691
689 692 args = decorator.split()
690 693 doctest_type = args[1]
691 694 if doctest_type in doctests:
692 695 doctests[doctest_type](self, args, input_lines, found, submitted)
693 696 else:
694 697 e = "Invalid option to @doctest: {0}".format(doctest_type)
695 698 raise Exception(e)
696 699
697 700
698 701 class IPythonDirective(Directive):
699 702
700 703 has_content = True
701 704 required_arguments = 0
702 705 optional_arguments = 4 # python, suppress, verbatim, doctest
703 706 final_argumuent_whitespace = True
704 707 option_spec = { 'python': directives.unchanged,
705 708 'suppress' : directives.flag,
706 709 'verbatim' : directives.flag,
707 710 'doctest' : directives.flag,
708 711 'okexcept': directives.flag,
709 712 'okwarning': directives.flag
710 713 }
711 714
712 715 shell = None
713 716
714 717 seen_docs = set()
715 718
716 719 def get_config_options(self):
717 720 # contains sphinx configuration variables
718 721 config = self.state.document.settings.env.config
719 722
720 723 # get config variables to set figure output directory
721 724 confdir = self.state.document.settings.env.app.confdir
722 725 savefig_dir = config.ipython_savefig_dir
723 726 source_dir = os.path.dirname(self.state.document.current_source)
724 727 if savefig_dir is None:
725 728 savefig_dir = config.html_static_path
726 729 if isinstance(savefig_dir, list):
727 730 savefig_dir = savefig_dir[0] # safe to assume only one path?
728 731 savefig_dir = os.path.join(confdir, savefig_dir)
729 732
730 733 # get regex and prompt stuff
731 734 rgxin = config.ipython_rgxin
732 735 rgxout = config.ipython_rgxout
733 736 promptin = config.ipython_promptin
734 737 promptout = config.ipython_promptout
735 738 mplbackend = config.ipython_mplbackend
736 739 exec_lines = config.ipython_execlines
737 740 hold_count = config.ipython_holdcount
738 741
739 742 return (savefig_dir, source_dir, rgxin, rgxout,
740 743 promptin, promptout, mplbackend, exec_lines, hold_count)
741 744
742 745 def setup(self):
743 746 # Get configuration values.
744 747 (savefig_dir, source_dir, rgxin, rgxout, promptin, promptout,
745 748 mplbackend, exec_lines, hold_count) = self.get_config_options()
746 749
747 750 if self.shell is None:
748 751 # We will be here many times. However, when the
749 752 # EmbeddedSphinxShell is created, its interactive shell member
750 753 # is the same for each instance.
751 754
752 755 if mplbackend:
753 756 import matplotlib
754 757 # Repeated calls to use() will not hurt us since `mplbackend`
755 758 # is the same each time.
756 759 matplotlib.use(mplbackend)
757 760
758 761 # Must be called after (potentially) importing matplotlib and
759 762 # setting its backend since exec_lines might import pylab.
760 763 self.shell = EmbeddedSphinxShell(exec_lines)
761 764
762 765 # Store IPython directive to enable better error messages
763 766 self.shell.directive = self
764 767
765 768 # reset the execution count if we haven't processed this doc
766 769 #NOTE: this may be borked if there are multiple seen_doc tmp files
767 770 #check time stamp?
768 771 if not self.state.document.current_source in self.seen_docs:
769 772 self.shell.IP.history_manager.reset()
770 773 self.shell.IP.execution_count = 1
771 774 self.shell.IP.prompt_manager.width = 0
772 775 self.seen_docs.add(self.state.document.current_source)
773 776
774 777 # and attach to shell so we don't have to pass them around
775 778 self.shell.rgxin = rgxin
776 779 self.shell.rgxout = rgxout
777 780 self.shell.promptin = promptin
778 781 self.shell.promptout = promptout
779 782 self.shell.savefig_dir = savefig_dir
780 783 self.shell.source_dir = source_dir
781 784 self.shell.hold_count = hold_count
782 785
783 786 # setup bookmark for saving figures directory
784 787 self.shell.process_input_line('bookmark ipy_savedir %s'%savefig_dir,
785 788 store_history=False)
786 789 self.shell.clear_cout()
787 790
788 791 return rgxin, rgxout, promptin, promptout
789 792
790 793 def teardown(self):
791 794 # delete last bookmark
792 795 self.shell.process_input_line('bookmark -d ipy_savedir',
793 796 store_history=False)
794 797 self.shell.clear_cout()
795 798
796 799 def run(self):
797 800 debug = False
798 801
799 802 #TODO, any reason block_parser can't be a method of embeddable shell
800 803 # then we wouldn't have to carry these around
801 804 rgxin, rgxout, promptin, promptout = self.setup()
802 805
803 806 options = self.options
804 807 self.shell.is_suppress = 'suppress' in options
805 808 self.shell.is_doctest = 'doctest' in options
806 809 self.shell.is_verbatim = 'verbatim' in options
807 810 self.shell.is_okexcept = 'okexcept' in options
808 811 self.shell.is_okwarning = 'okwarning' in options
809 812
810 813 # handle pure python code
811 814 if 'python' in self.arguments:
812 815 content = self.content
813 816 self.content = self.shell.process_pure_python(content)
814 817
815 818 parts = '\n'.join(self.content).split('\n\n')
816 819
817 820 lines = ['.. code-block:: ipython', '']
818 821 figures = []
819 822
820 823 for part in parts:
821 824 block = block_parser(part, rgxin, rgxout, promptin, promptout)
822 825 if len(block):
823 826 rows, figure = self.shell.process_block(block)
824 827 for row in rows:
825 828 lines.extend([' %s'%line for line in row.split('\n')])
826 829
827 830 if figure is not None:
828 831 figures.append(figure)
829 832
830 833 for figure in figures:
831 834 lines.append('')
832 835 lines.extend(figure.split('\n'))
833 836 lines.append('')
834 837
835 838 if len(lines)>2:
836 839 if debug:
837 840 print('\n'.join(lines))
838 841 else:
839 842 # This has to do with input, not output. But if we comment
840 843 # these lines out, then no IPython code will appear in the
841 844 # final output.
842 845 self.state_machine.insert_input(
843 846 lines, self.state_machine.input_lines.source(0))
844 847
845 848 # cleanup
846 849 self.teardown()
847 850
848 851 return []
849 852
850 853 # Enable as a proper Sphinx directive
851 854 def setup(app):
852 855 setup.app = app
853 856
854 857 app.add_directive('ipython', IPythonDirective)
855 858 app.add_config_value('ipython_savefig_dir', None, 'env')
856 859 app.add_config_value('ipython_rgxin',
857 860 re.compile('In \[(\d+)\]:\s?(.*)\s*'), 'env')
858 861 app.add_config_value('ipython_rgxout',
859 862 re.compile('Out\[(\d+)\]:\s?(.*)\s*'), 'env')
860 863 app.add_config_value('ipython_promptin', 'In [%d]:', 'env')
861 864 app.add_config_value('ipython_promptout', 'Out[%d]:', 'env')
862 865
863 866 # We could just let matplotlib pick whatever is specified as the default
864 867 # backend in the matplotlibrc file, but this would cause issues if the
865 868 # backend didn't work in headless environments. For this reason, 'agg'
866 869 # is a good default backend choice.
867 870 app.add_config_value('ipython_mplbackend', 'agg', 'env')
868 871
869 872 # If the user sets this config value to `None`, then EmbeddedSphinxShell's
870 873 # __init__ method will treat it as [].
871 874 execlines = ['import numpy as np', 'import matplotlib.pyplot as plt']
872 875 app.add_config_value('ipython_execlines', execlines, 'env')
873 876
874 877 app.add_config_value('ipython_holdcount', True, 'env')
875 878
876 879 # Simple smoke test, needs to be converted to a proper automatic test.
877 880 def test():
878 881
879 882 examples = [
880 883 r"""
881 884 In [9]: pwd
882 885 Out[9]: '/home/jdhunter/py4science/book'
883 886
884 887 In [10]: cd bookdata/
885 888 /home/jdhunter/py4science/book/bookdata
886 889
887 890 In [2]: from pylab import *
888 891
889 892 In [2]: ion()
890 893
891 894 In [3]: im = imread('stinkbug.png')
892 895
893 896 @savefig mystinkbug.png width=4in
894 897 In [4]: imshow(im)
895 898 Out[4]: <matplotlib.image.AxesImage object at 0x39ea850>
896 899
897 900 """,
898 901 r"""
899 902
900 903 In [1]: x = 'hello world'
901 904
902 905 # string methods can be
903 906 # used to alter the string
904 907 @doctest
905 908 In [2]: x.upper()
906 909 Out[2]: 'HELLO WORLD'
907 910
908 911 @verbatim
909 912 In [3]: x.st<TAB>
910 913 x.startswith x.strip
911 914 """,
912 915 r"""
913 916
914 917 In [130]: url = 'http://ichart.finance.yahoo.com/table.csv?s=CROX\
915 918 .....: &d=9&e=22&f=2009&g=d&a=1&br=8&c=2006&ignore=.csv'
916 919
917 920 In [131]: print url.split('&')
918 921 ['http://ichart.finance.yahoo.com/table.csv?s=CROX', 'd=9', 'e=22', 'f=2009', 'g=d', 'a=1', 'b=8', 'c=2006', 'ignore=.csv']
919 922
920 923 In [60]: import urllib
921 924
922 925 """,
923 926 r"""\
924 927
925 928 In [133]: import numpy.random
926 929
927 930 @suppress
928 931 In [134]: numpy.random.seed(2358)
929 932
930 933 @doctest
931 934 In [135]: numpy.random.rand(10,2)
932 935 Out[135]:
933 936 array([[ 0.64524308, 0.59943846],
934 937 [ 0.47102322, 0.8715456 ],
935 938 [ 0.29370834, 0.74776844],
936 939 [ 0.99539577, 0.1313423 ],
937 940 [ 0.16250302, 0.21103583],
938 941 [ 0.81626524, 0.1312433 ],
939 942 [ 0.67338089, 0.72302393],
940 943 [ 0.7566368 , 0.07033696],
941 944 [ 0.22591016, 0.77731835],
942 945 [ 0.0072729 , 0.34273127]])
943 946
944 947 """,
945 948
946 949 r"""
947 950 In [106]: print x
948 951 jdh
949 952
950 953 In [109]: for i in range(10):
951 954 .....: print i
952 955 .....:
953 956 .....:
954 957 0
955 958 1
956 959 2
957 960 3
958 961 4
959 962 5
960 963 6
961 964 7
962 965 8
963 966 9
964 967 """,
965 968
966 969 r"""
967 970
968 971 In [144]: from pylab import *
969 972
970 973 In [145]: ion()
971 974
972 975 # use a semicolon to suppress the output
973 976 @savefig test_hist.png width=4in
974 977 In [151]: hist(np.random.randn(10000), 100);
975 978
976 979
977 980 @savefig test_plot.png width=4in
978 981 In [151]: plot(np.random.randn(10000), 'o');
979 982 """,
980 983
981 984 r"""
982 985 # use a semicolon to suppress the output
983 986 In [151]: plt.clf()
984 987
985 988 @savefig plot_simple.png width=4in
986 989 In [151]: plot([1,2,3])
987 990
988 991 @savefig hist_simple.png width=4in
989 992 In [151]: hist(np.random.randn(10000), 100);
990 993
991 994 """,
992 995 r"""
993 996 # update the current fig
994 997 In [151]: ylabel('number')
995 998
996 999 In [152]: title('normal distribution')
997 1000
998 1001
999 1002 @savefig hist_with_text.png
1000 1003 In [153]: grid(True)
1001 1004
1002 1005 @doctest float
1003 1006 In [154]: 0.1 + 0.2
1004 1007 Out[154]: 0.3
1005 1008
1006 1009 @doctest float
1007 1010 In [155]: np.arange(16).reshape(4,4)
1008 1011 Out[155]:
1009 1012 array([[ 0, 1, 2, 3],
1010 1013 [ 4, 5, 6, 7],
1011 1014 [ 8, 9, 10, 11],
1012 1015 [12, 13, 14, 15]])
1013 1016
1014 1017 In [1]: x = np.arange(16, dtype=float).reshape(4,4)
1015 1018
1016 1019 In [2]: x[0,0] = np.inf
1017 1020
1018 1021 In [3]: x[0,1] = np.nan
1019 1022
1020 1023 @doctest float
1021 1024 In [4]: x
1022 1025 Out[4]:
1023 1026 array([[ inf, nan, 2., 3.],
1024 1027 [ 4., 5., 6., 7.],
1025 1028 [ 8., 9., 10., 11.],
1026 1029 [ 12., 13., 14., 15.]])
1027 1030
1028 1031
1029 1032 """,
1030 1033 ]
1031 1034 # skip local-file depending first example:
1032 1035 examples = examples[1:]
1033 1036
1034 1037 #ipython_directive.DEBUG = True # dbg
1035 1038 #options = dict(suppress=True) # dbg
1036 1039 options = dict()
1037 1040 for example in examples:
1038 1041 content = example.split('\n')
1039 1042 IPythonDirective('debug', arguments=None, options=options,
1040 1043 content=content, lineno=0,
1041 1044 content_offset=None, block_text=None,
1042 1045 state=None, state_machine=None,
1043 1046 )
1044 1047
1045 1048 # Run test suite as a script
1046 1049 if __name__=='__main__':
1047 1050 if not os.path.isdir('_static'):
1048 1051 os.mkdir('_static')
1049 1052 test()
1050 1053 print('All OK? Check figures in _static/')
General Comments 0
You need to be logged in to leave comments. Login now