Show More
This diff has been collapsed as it changes many lines, (507 lines changed) Show them Hide them | |||
@@ -0,0 +1,507 b'' | |||
|
1 | /** | |
|
2 | * Alternate Sphinx design | |
|
3 | * Originally created by Armin Ronacher for Werkzeug, adapted by Georg Brandl. | |
|
4 | */ | |
|
5 | ||
|
6 | body { | |
|
7 | font-family: 'Lucida Grande', 'Lucida Sans Unicode', 'Geneva', 'Verdana', sans-serif; | |
|
8 | font-size: 14px; | |
|
9 | letter-spacing: -0.01em; | |
|
10 | line-height: 150%; | |
|
11 | text-align: center; | |
|
12 | /*background-color: #AFC1C4; */ | |
|
13 | background-color: #BFD1D4; | |
|
14 | color: black; | |
|
15 | padding: 0; | |
|
16 | border: 1px solid #aaa; | |
|
17 | ||
|
18 | margin: 0px 80px 0px 80px; | |
|
19 | min-width: 740px; | |
|
20 | } | |
|
21 | ||
|
22 | a { | |
|
23 | color: #CA7900; | |
|
24 | text-decoration: none; | |
|
25 | } | |
|
26 | ||
|
27 | a:hover { | |
|
28 | color: #2491CF; | |
|
29 | } | |
|
30 | ||
|
31 | pre { | |
|
32 | font-family: 'Consolas', 'Deja Vu Sans Mono', 'Bitstream Vera Sans Mono', monospace; | |
|
33 | font-size: 0.95em; | |
|
34 | letter-spacing: 0.015em; | |
|
35 | padding: 0.5em; | |
|
36 | border: 1px solid #ccc; | |
|
37 | background-color: #f8f8f8; | |
|
38 | } | |
|
39 | ||
|
40 | td.linenos pre { | |
|
41 | padding: 0.5em 0; | |
|
42 | border: 0; | |
|
43 | background-color: transparent; | |
|
44 | color: #aaa; | |
|
45 | } | |
|
46 | ||
|
47 | table.highlighttable { | |
|
48 | margin-left: 0.5em; | |
|
49 | } | |
|
50 | ||
|
51 | table.highlighttable td { | |
|
52 | padding: 0 0.5em 0 0.5em; | |
|
53 | } | |
|
54 | ||
|
55 | cite, code, tt { | |
|
56 | font-family: 'Consolas', 'Deja Vu Sans Mono', 'Bitstream Vera Sans Mono', monospace; | |
|
57 | font-size: 0.95em; | |
|
58 | letter-spacing: 0.01em; | |
|
59 | } | |
|
60 | ||
|
61 | hr { | |
|
62 | border: 1px solid #abc; | |
|
63 | margin: 2em; | |
|
64 | } | |
|
65 | ||
|
66 | tt { | |
|
67 | background-color: #f2f2f2; | |
|
68 | border-bottom: 1px solid #ddd; | |
|
69 | color: #333; | |
|
70 | } | |
|
71 | ||
|
72 | tt.descname { | |
|
73 | background-color: transparent; | |
|
74 | font-weight: bold; | |
|
75 | font-size: 1.2em; | |
|
76 | border: 0; | |
|
77 | } | |
|
78 | ||
|
79 | tt.descclassname { | |
|
80 | background-color: transparent; | |
|
81 | border: 0; | |
|
82 | } | |
|
83 | ||
|
84 | tt.xref { | |
|
85 | background-color: transparent; | |
|
86 | font-weight: bold; | |
|
87 | border: 0; | |
|
88 | } | |
|
89 | ||
|
90 | a tt { | |
|
91 | background-color: transparent; | |
|
92 | font-weight: bold; | |
|
93 | border: 0; | |
|
94 | color: #CA7900; | |
|
95 | } | |
|
96 | ||
|
97 | a tt:hover { | |
|
98 | color: #2491CF; | |
|
99 | } | |
|
100 | ||
|
101 | dl { | |
|
102 | margin-bottom: 15px; | |
|
103 | } | |
|
104 | ||
|
105 | dd p { | |
|
106 | margin-top: 0px; | |
|
107 | } | |
|
108 | ||
|
109 | dd ul, dd table { | |
|
110 | margin-bottom: 10px; | |
|
111 | } | |
|
112 | ||
|
113 | dd { | |
|
114 | margin-top: 3px; | |
|
115 | margin-bottom: 10px; | |
|
116 | margin-left: 30px; | |
|
117 | } | |
|
118 | ||
|
119 | .refcount { | |
|
120 | color: #060; | |
|
121 | } | |
|
122 | ||
|
123 | dt:target, | |
|
124 | .highlight { | |
|
125 | background-color: #fbe54e; | |
|
126 | } | |
|
127 | ||
|
128 | dl.class, dl.function { | |
|
129 | border-top: 2px solid #888; | |
|
130 | } | |
|
131 | ||
|
132 | dl.method, dl.attribute { | |
|
133 | border-top: 1px solid #aaa; | |
|
134 | } | |
|
135 | ||
|
136 | dl.glossary dt { | |
|
137 | font-weight: bold; | |
|
138 | font-size: 1.1em; | |
|
139 | } | |
|
140 | ||
|
141 | pre { | |
|
142 | line-height: 120%; | |
|
143 | } | |
|
144 | ||
|
145 | pre a { | |
|
146 | color: inherit; | |
|
147 | text-decoration: underline; | |
|
148 | } | |
|
149 | ||
|
150 | .first { | |
|
151 | margin-top: 0 !important; | |
|
152 | } | |
|
153 | ||
|
154 | div.document { | |
|
155 | background-color: white; | |
|
156 | text-align: left; | |
|
157 | background-image: url(contents.png); | |
|
158 | background-repeat: repeat-x; | |
|
159 | } | |
|
160 | ||
|
161 | /* | |
|
162 | div.documentwrapper { | |
|
163 | width: 100%; | |
|
164 | } | |
|
165 | */ | |
|
166 | ||
|
167 | div.clearer { | |
|
168 | clear: both; | |
|
169 | } | |
|
170 | ||
|
171 | div.related h3 { | |
|
172 | display: none; | |
|
173 | } | |
|
174 | ||
|
175 | div.related ul { | |
|
176 | background-image: url(navigation.png); | |
|
177 | height: 2em; | |
|
178 | list-style: none; | |
|
179 | border-top: 1px solid #ddd; | |
|
180 | border-bottom: 1px solid #ddd; | |
|
181 | margin: 0; | |
|
182 | padding-left: 10px; | |
|
183 | } | |
|
184 | ||
|
185 | div.related ul li { | |
|
186 | margin: 0; | |
|
187 | padding: 0; | |
|
188 | height: 2em; | |
|
189 | float: left; | |
|
190 | } | |
|
191 | ||
|
192 | div.related ul li.right { | |
|
193 | float: right; | |
|
194 | margin-right: 5px; | |
|
195 | } | |
|
196 | ||
|
197 | div.related ul li a { | |
|
198 | margin: 0; | |
|
199 | padding: 0 5px 0 5px; | |
|
200 | line-height: 1.75em; | |
|
201 | color: #EE9816; | |
|
202 | } | |
|
203 | ||
|
204 | div.related ul li a:hover { | |
|
205 | color: #3CA8E7; | |
|
206 | } | |
|
207 | ||
|
208 | div.body { | |
|
209 | margin: 0; | |
|
210 | padding: 0.5em 20px 20px 20px; | |
|
211 | } | |
|
212 | ||
|
213 | div.bodywrapper { | |
|
214 | margin: 0 240px 0 0; | |
|
215 | border-right: 1px solid #ccc; | |
|
216 | } | |
|
217 | ||
|
218 | div.body a { | |
|
219 | text-decoration: underline; | |
|
220 | } | |
|
221 | ||
|
222 | div.sphinxsidebar { | |
|
223 | margin: 0; | |
|
224 | padding: 0.5em 15px 15px 0; | |
|
225 | width: 210px; | |
|
226 | float: right; | |
|
227 | text-align: left; | |
|
228 | /* margin-left: -100%; */ | |
|
229 | } | |
|
230 | ||
|
231 | div.sphinxsidebar h4, div.sphinxsidebar h3 { | |
|
232 | margin: 1em 0 0.5em 0; | |
|
233 | font-size: 0.9em; | |
|
234 | padding: 0.1em 0 0.1em 0.5em; | |
|
235 | color: white; | |
|
236 | border: 1px solid #86989B; | |
|
237 | background-color: #AFC1C4; | |
|
238 | } | |
|
239 | ||
|
240 | div.sphinxsidebar ul { | |
|
241 | padding-left: 1.5em; | |
|
242 | margin-top: 7px; | |
|
243 | list-style: none; | |
|
244 | padding: 0; | |
|
245 | line-height: 130%; | |
|
246 | } | |
|
247 | ||
|
248 | div.sphinxsidebar ul ul { | |
|
249 | list-style: square; | |
|
250 | margin-left: 20px; | |
|
251 | } | |
|
252 | ||
|
253 | p { | |
|
254 | margin: 0.8em 0 0.5em 0; | |
|
255 | } | |
|
256 | ||
|
257 | p.rubric { | |
|
258 | font-weight: bold; | |
|
259 | } | |
|
260 | ||
|
261 | h1 { | |
|
262 | margin: 0; | |
|
263 | padding: 0.7em 0 0.3em 0; | |
|
264 | font-size: 1.5em; | |
|
265 | color: #11557C; | |
|
266 | } | |
|
267 | ||
|
268 | h2 { | |
|
269 | margin: 1.3em 0 0.2em 0; | |
|
270 | font-size: 1.35em; | |
|
271 | padding: 0; | |
|
272 | } | |
|
273 | ||
|
274 | h3 { | |
|
275 | margin: 1em 0 -0.3em 0; | |
|
276 | font-size: 1.2em; | |
|
277 | } | |
|
278 | ||
|
279 | h1 a, h2 a, h3 a, h4 a, h5 a, h6 a { | |
|
280 | color: black!important; | |
|
281 | } | |
|
282 | ||
|
283 | h1 a.anchor, h2 a.anchor, h3 a.anchor, h4 a.anchor, h5 a.anchor, h6 a.anchor { | |
|
284 | display: none; | |
|
285 | margin: 0 0 0 0.3em; | |
|
286 | padding: 0 0.2em 0 0.2em; | |
|
287 | color: #aaa!important; | |
|
288 | } | |
|
289 | ||
|
290 | h1:hover a.anchor, h2:hover a.anchor, h3:hover a.anchor, h4:hover a.anchor, | |
|
291 | h5:hover a.anchor, h6:hover a.anchor { | |
|
292 | display: inline; | |
|
293 | } | |
|
294 | ||
|
295 | h1 a.anchor:hover, h2 a.anchor:hover, h3 a.anchor:hover, h4 a.anchor:hover, | |
|
296 | h5 a.anchor:hover, h6 a.anchor:hover { | |
|
297 | color: #777; | |
|
298 | background-color: #eee; | |
|
299 | } | |
|
300 | ||
|
301 | table { | |
|
302 | border-collapse: collapse; | |
|
303 | margin: 0 -0.5em 0 -0.5em; | |
|
304 | } | |
|
305 | ||
|
306 | table td, table th { | |
|
307 | padding: 0.2em 0.5em 0.2em 0.5em; | |
|
308 | } | |
|
309 | ||
|
310 | div.footer { | |
|
311 | background-color: #E3EFF1; | |
|
312 | color: #86989B; | |
|
313 | padding: 3px 8px 3px 0; | |
|
314 | clear: both; | |
|
315 | font-size: 0.8em; | |
|
316 | text-align: right; | |
|
317 | } | |
|
318 | ||
|
319 | div.footer a { | |
|
320 | color: #86989B; | |
|
321 | text-decoration: underline; | |
|
322 | } | |
|
323 | ||
|
324 | div.pagination { | |
|
325 | margin-top: 2em; | |
|
326 | padding-top: 0.5em; | |
|
327 | border-top: 1px solid black; | |
|
328 | text-align: center; | |
|
329 | } | |
|
330 | ||
|
331 | div.sphinxsidebar ul.toc { | |
|
332 | margin: 1em 0 1em 0; | |
|
333 | padding: 0 0 0 0.5em; | |
|
334 | list-style: none; | |
|
335 | } | |
|
336 | ||
|
337 | div.sphinxsidebar ul.toc li { | |
|
338 | margin: 0.5em 0 0.5em 0; | |
|
339 | font-size: 0.9em; | |
|
340 | line-height: 130%; | |
|
341 | } | |
|
342 | ||
|
343 | div.sphinxsidebar ul.toc li p { | |
|
344 | margin: 0; | |
|
345 | padding: 0; | |
|
346 | } | |
|
347 | ||
|
348 | div.sphinxsidebar ul.toc ul { | |
|
349 | margin: 0.2em 0 0.2em 0; | |
|
350 | padding: 0 0 0 1.8em; | |
|
351 | } | |
|
352 | ||
|
353 | div.sphinxsidebar ul.toc ul li { | |
|
354 | padding: 0; | |
|
355 | } | |
|
356 | ||
|
357 | div.admonition, div.warning { | |
|
358 | font-size: 0.9em; | |
|
359 | margin: 1em 0 0 0; | |
|
360 | border: 1px solid #86989B; | |
|
361 | background-color: #f7f7f7; | |
|
362 | } | |
|
363 | ||
|
364 | div.admonition p, div.warning p { | |
|
365 | margin: 0.5em 1em 0.5em 1em; | |
|
366 | padding: 0; | |
|
367 | } | |
|
368 | ||
|
369 | div.admonition pre, div.warning pre { | |
|
370 | margin: 0.4em 1em 0.4em 1em; | |
|
371 | } | |
|
372 | ||
|
373 | div.admonition p.admonition-title, | |
|
374 | div.warning p.admonition-title { | |
|
375 | margin: 0; | |
|
376 | padding: 0.1em 0 0.1em 0.5em; | |
|
377 | color: white; | |
|
378 | border-bottom: 1px solid #86989B; | |
|
379 | font-weight: bold; | |
|
380 | background-color: #AFC1C4; | |
|
381 | } | |
|
382 | ||
|
383 | div.warning { | |
|
384 | border: 1px solid #940000; | |
|
385 | } | |
|
386 | ||
|
387 | div.warning p.admonition-title { | |
|
388 | background-color: #CF0000; | |
|
389 | border-bottom-color: #940000; | |
|
390 | } | |
|
391 | ||
|
392 | div.admonition ul, div.admonition ol, | |
|
393 | div.warning ul, div.warning ol { | |
|
394 | margin: 0.1em 0.5em 0.5em 3em; | |
|
395 | padding: 0; | |
|
396 | } | |
|
397 | ||
|
398 | div.versioninfo { | |
|
399 | margin: 1em 0 0 0; | |
|
400 | border: 1px solid #ccc; | |
|
401 | background-color: #DDEAF0; | |
|
402 | padding: 8px; | |
|
403 | line-height: 1.3em; | |
|
404 | font-size: 0.9em; | |
|
405 | } | |
|
406 | ||
|
407 | ||
|
408 | a.headerlink { | |
|
409 | color: #c60f0f!important; | |
|
410 | font-size: 1em; | |
|
411 | margin-left: 6px; | |
|
412 | padding: 0 4px 0 4px; | |
|
413 | text-decoration: none!important; | |
|
414 | visibility: hidden; | |
|
415 | } | |
|
416 | ||
|
417 | h1:hover > a.headerlink, | |
|
418 | h2:hover > a.headerlink, | |
|
419 | h3:hover > a.headerlink, | |
|
420 | h4:hover > a.headerlink, | |
|
421 | h5:hover > a.headerlink, | |
|
422 | h6:hover > a.headerlink, | |
|
423 | dt:hover > a.headerlink { | |
|
424 | visibility: visible; | |
|
425 | } | |
|
426 | ||
|
427 | a.headerlink:hover { | |
|
428 | background-color: #ccc; | |
|
429 | color: white!important; | |
|
430 | } | |
|
431 | ||
|
432 | table.indextable td { | |
|
433 | text-align: left; | |
|
434 | vertical-align: top; | |
|
435 | } | |
|
436 | ||
|
437 | table.indextable dl, table.indextable dd { | |
|
438 | margin-top: 0; | |
|
439 | margin-bottom: 0; | |
|
440 | } | |
|
441 | ||
|
442 | table.indextable tr.pcap { | |
|
443 | height: 10px; | |
|
444 | } | |
|
445 | ||
|
446 | table.indextable tr.cap { | |
|
447 | margin-top: 10px; | |
|
448 | background-color: #f2f2f2; | |
|
449 | } | |
|
450 | ||
|
451 | img.toggler { | |
|
452 | margin-right: 3px; | |
|
453 | margin-top: 3px; | |
|
454 | cursor: pointer; | |
|
455 | } | |
|
456 | ||
|
457 | img.inheritance { | |
|
458 | border: 0px | |
|
459 | } | |
|
460 | ||
|
461 | form.pfform { | |
|
462 | margin: 10px 0 20px 0; | |
|
463 | } | |
|
464 | ||
|
465 | table.contentstable { | |
|
466 | width: 90%; | |
|
467 | } | |
|
468 | ||
|
469 | table.contentstable p.biglink { | |
|
470 | line-height: 150%; | |
|
471 | } | |
|
472 | ||
|
473 | a.biglink { | |
|
474 | font-size: 1.3em; | |
|
475 | } | |
|
476 | ||
|
477 | span.linkdescr { | |
|
478 | font-style: italic; | |
|
479 | padding-top: 5px; | |
|
480 | font-size: 90%; | |
|
481 | } | |
|
482 | ||
|
483 | ul.search { | |
|
484 | margin: 10px 0 0 20px; | |
|
485 | padding: 0; | |
|
486 | } | |
|
487 | ||
|
488 | ul.search li { | |
|
489 | padding: 5px 0 5px 20px; | |
|
490 | background-image: url(file.png); | |
|
491 | background-repeat: no-repeat; | |
|
492 | background-position: 0 7px; | |
|
493 | } | |
|
494 | ||
|
495 | ul.search li a { | |
|
496 | font-weight: bold; | |
|
497 | } | |
|
498 | ||
|
499 | ul.search li div.context { | |
|
500 | color: #888; | |
|
501 | margin: 2px 0 0 30px; | |
|
502 | text-align: left; | |
|
503 | } | |
|
504 | ||
|
505 | ul.keywordmatches li.goodmatch a { | |
|
506 | font-weight: bold; | |
|
507 | } |
@@ -0,0 +1,23 b'' | |||
|
1 | {% extends "!layout.html" %} | |
|
2 | ||
|
3 | ||
|
4 | {% block rootrellink %} | |
|
5 | <li><a href="{{ pathto('index') }}">home</a>| </li> | |
|
6 | <li><a href="{{ pathto('search') }}">search</a>| </li> | |
|
7 | <li><a href="{{ pathto('whatsnew/index') }}">what's new </a> »</li> | |
|
8 | {% endblock %} | |
|
9 | ||
|
10 | ||
|
11 | {% block relbar1 %} | |
|
12 | ||
|
13 | <div style="background-color: white; text-align: left; padding: 10px 10px 15px 15px"> | |
|
14 | <a href="{{ pathto('index') }}"><img src="{{ | |
|
15 | pathto("_static/logo.png", 1) }}" border="0" alt="IPython Documentation"/></a> | |
|
16 | </div> | |
|
17 | {{ super() }} | |
|
18 | {% endblock %} | |
|
19 | ||
|
20 | {# put the sidebar before the body #} | |
|
21 | {% block sidebar1 %}{{ sidebar() }}{% endblock %} | |
|
22 | {% block sidebar2 %}{% endblock %} | |
|
23 |
@@ -0,0 +1,13 b'' | |||
|
1 | .. _about_index: | |
|
2 | ||
|
3 | ============= | |
|
4 | About IPython | |
|
5 | ============= | |
|
6 | ||
|
7 | .. toctree:: | |
|
8 | :maxdepth: 1 | |
|
9 | ||
|
10 | credits | |
|
11 | history | |
|
12 | license_and_copyright | |
|
13 |
@@ -0,0 +1,49 b'' | |||
|
1 | ================================================ | |
|
2 | Development version | |
|
3 | ================================================ | |
|
4 | ||
|
5 | Main `ipython` branch | |
|
6 | ===================== | |
|
7 | ||
|
8 | New features | |
|
9 | ------------ | |
|
10 | ||
|
11 | * Added a new module :mod:`IPython.lib.inputhook` to manage the integration | |
|
12 | with GUI event loops using `PyOS_InputHook`. See the docstrings in this | |
|
13 | module or the main IPython docs for details. | |
|
14 | * For users, GUI event loop integration is now handled through the new | |
|
15 | :command:`%gui` magic command. Type ``%gui?`` at an IPython prompt for | |
|
16 | documentation. | |
|
17 | * For developers :mod:`IPython.lib.inputhook` provides a simple interface | |
|
18 | for managing the event loops in their interactive GUI applications. | |
|
19 | Examples can be found in our :file:`docs/examples/lib` directory. | |
|
20 | ||
|
21 | Bug fixes | |
|
22 | --------- | |
|
23 | ||
|
24 | * Keyboard interrupts now work with GUI support enabled across all platforms | |
|
25 | and all GUI toolkits reliably. | |
|
26 | ||
|
27 | Backwards incompatible changes | |
|
28 | ------------------------------ | |
|
29 | ||
|
30 | * Support for ``qt3`` has been dropped. User's who need this should use | |
|
31 | previous versions of IPython. | |
|
32 | * Removed :mod:`shellglobals` as it was obsolete. | |
|
33 | * Removed all the threaded shells in :mod:`IPython.core.shell`. These are no | |
|
34 | longer needed because of the new capabilities in | |
|
35 | :mod:`IPython.lib.inputhook`. | |
|
36 | * The old threading command line flags (pylab/wthread/etc.) have been | |
|
37 | deprecated. Use :mod:`IPython.inputhook` or the new :command:`%gui` magic | |
|
38 | command instead. | |
|
39 | * New top-level sub-packages have been created: :mod:`IPython.core`, | |
|
40 | :mod:`IPython.lib`, :mod:`IPython.utils`, :mod:`IPython.deathrow`, | |
|
41 | :mod:`IPython.quarantine`. All existing top-level modules have been | |
|
42 | moved to appropriate sub-packages. All internal import statements | |
|
43 | have been updated and tests have been added. The build system (setup.py | |
|
44 | and friends) have been updated. | |
|
45 | * Compatability modules have been created for :mod:`IPython.Shell`, | |
|
46 | :mod:`IPython.ipapi` and :mod:`IPython.iplib` that display warnings | |
|
47 | and then load the actual implementation from :mod:`IPython.core`. | |
|
48 | * :mod:`Extensions` has been moved to :mod:`extensions`. | |
|
49 |
@@ -0,0 +1,23 b'' | |||
|
1 | .. Developers should add in this file, during each release cycle, information | |
|
2 | .. about important changes they've made, in a summary format that's meant for | |
|
3 | .. end users. For each release we normally have three sections: features, bug | |
|
4 | .. fixes and api breakage. | |
|
5 | .. Please remember to credit the authors of the contributions by name, | |
|
6 | .. especially when they are new users or developers who do not regularly | |
|
7 | .. participate in IPython's development. | |
|
8 | ||
|
9 | .. _whatsnew_index: | |
|
10 | ||
|
11 | ===================== | |
|
12 | What's new in IPython | |
|
13 | ===================== | |
|
14 | ||
|
15 | .. toctree:: | |
|
16 | :maxdepth: 1 | |
|
17 | ||
|
18 | development | |
|
19 | version0.10 | |
|
20 | version0.9 | |
|
21 | version0.8 | |
|
22 | ||
|
23 |
@@ -0,0 +1,223 b'' | |||
|
1 | ======================================== | |
|
2 | 0.10 series | |
|
3 | ======================================== | |
|
4 | ||
|
5 | Release 0.10 | |
|
6 | ============ | |
|
7 | ||
|
8 | This release brings months of slow but steady development, and will be the last | |
|
9 | before a major restructuring and cleanup of IPython's internals that is already | |
|
10 | under way. For this reason, we hope that 0.10 will be a stable and robust | |
|
11 | release so that while users adapt to some of the API changes that will come | |
|
12 | with the refactoring that will become IPython 0.11, they can safely use 0.10 in | |
|
13 | all existing projects with minimal changes (if any). | |
|
14 | ||
|
15 | IPython 0.10 is now a medium-sized project, with roughly (as reported by David | |
|
16 | Wheeler's :command:`sloccount` utility) 40750 lines of Python code, and a diff | |
|
17 | between 0.9.1 and this release that contains almost 28000 lines of code and | |
|
18 | documentation. Our documentation, in PDF format, is a 495-page long PDF | |
|
19 | document (also available in HTML format, both generated from the same sources). | |
|
20 | ||
|
21 | Many users and developers contributed code, features, bug reports and ideas to | |
|
22 | this release. Please do not hesitate in contacting us if we've failed to | |
|
23 | acknowledge your contribution here. In particular, for this release we have | |
|
24 | contribution from the following people, a mix of new and regular names (in | |
|
25 | alphabetical order by first name): | |
|
26 | ||
|
27 | * Alexander Clausen: fix #341726. | |
|
28 | * Brian Granger: lots of work everywhere (features, bug fixes, etc). | |
|
29 | * Daniel Ashbrook: bug report on MemoryError during compilation, now fixed. | |
|
30 | * Darren Dale: improvements to documentation build system, feedback, design | |
|
31 | ideas. | |
|
32 | * Fernando Perez: various places. | |
|
33 | * Gaël Varoquaux: core code, ipythonx GUI, design discussions, etc. Lots... | |
|
34 | * John Hunter: suggestions, bug fixes, feedback. | |
|
35 | * Jorgen Stenarson: work on many fronts, tests, fixes, win32 support, etc. | |
|
36 | * Laurent Dufréchou: many improvements to ipython-wx standalone app. | |
|
37 | * Lukasz Pankowski: prefilter, `%edit`, demo improvements. | |
|
38 | * Matt Foster: TextMate support in `%edit`. | |
|
39 | * Nathaniel Smith: fix #237073. | |
|
40 | * Pauli Virtanen: fixes and improvements to extensions, documentation. | |
|
41 | * Prabhu Ramachandran: improvements to `%timeit`. | |
|
42 | * Robert Kern: several extensions. | |
|
43 | * Sameer D'Costa: help on critical bug #269966. | |
|
44 | * Stephan Peijnik: feedback on Debian compliance and many man pages. | |
|
45 | * Steven Bethard: we are now shipping his :mod:`argparse` module. | |
|
46 | * Tom Fetherston: many improvements to :mod:`IPython.demo` module. | |
|
47 | * Ville Vainio: lots of work everywhere (features, bug fixes, etc). | |
|
48 | * Vishal Vasta: ssh support in ipcluster. | |
|
49 | * Walter Doerwald: work on the :mod:`IPython.ipipe` system. | |
|
50 | ||
|
51 | Below we give an overview of new features, bug fixes and backwards-incompatible | |
|
52 | changes. For a detailed account of every change made, feel free to view the | |
|
53 | project log with :command:`bzr log`. | |
|
54 | ||
|
55 | New features | |
|
56 | ------------ | |
|
57 | ||
|
58 | * New `%paste` magic automatically extracts current contents of clipboard and | |
|
59 | pastes it directly, while correctly handling code that is indented or | |
|
60 | prepended with `>>>` or `...` python prompt markers. A very useful new | |
|
61 | feature contributed by Robert Kern. | |
|
62 | ||
|
63 | * IPython 'demos', created with the :mod:`IPython.demo` module, can now be | |
|
64 | created from files on disk or strings in memory. Other fixes and | |
|
65 | improvements to the demo system, by Tom Fetherston. | |
|
66 | ||
|
67 | * Added :func:`find_cmd` function to :mod:`IPython.platutils` module, to find | |
|
68 | commands in a cross-platform manner. | |
|
69 | ||
|
70 | * Many improvements and fixes to Gaël Varoquaux's :command:`ipythonx`, a | |
|
71 | WX-based lightweight IPython instance that can be easily embedded in other WX | |
|
72 | applications. These improvements have made it possible to now have an | |
|
73 | embedded IPython in Mayavi and other tools. | |
|
74 | ||
|
75 | * :class:`MultiengineClient` objects now have a :meth:`benchmark` method. | |
|
76 | ||
|
77 | * The manual now includes a full set of auto-generated API documents from the | |
|
78 | code sources, using Sphinx and some of our own support code. We are now | |
|
79 | using the `Numpy Documentation Standard`_ for all docstrings, and we have | |
|
80 | tried to update as many existing ones as possible to this format. | |
|
81 | ||
|
82 | * The new :mod:`IPython.Extensions.ipy_pretty` extension by Robert Kern | |
|
83 | provides configurable pretty-printing. | |
|
84 | ||
|
85 | * Many improvements to the :command:`ipython-wx` standalone WX-based IPython | |
|
86 | application by Laurent Dufréchou. It can optionally run in a thread, and | |
|
87 | this can be toggled at runtime (allowing the loading of Matplotlib in a | |
|
88 | running session without ill effects). | |
|
89 | ||
|
90 | * IPython includes a copy of Steven Bethard's argparse_ in the | |
|
91 | :mod:`IPython.external` package, so we can use it internally and it is also | |
|
92 | available to any IPython user. By installing it in this manner, we ensure | |
|
93 | zero conflicts with any system-wide installation you may already have while | |
|
94 | minimizing external dependencies for new users. In IPython 0.10, We ship | |
|
95 | argparse version 1.0. | |
|
96 | ||
|
97 | * An improved and much more robust test suite, that runs groups of tests in | |
|
98 | separate subprocesses using either Nose or Twisted's :command:`trial` runner | |
|
99 | to ensure proper management of Twisted-using code. The test suite degrades | |
|
100 | gracefully if optional dependencies are not available, so that the | |
|
101 | :command:`iptest` command can be run with only Nose installed and nothing | |
|
102 | else. We also have more and cleaner test decorators to better select tests | |
|
103 | depending on runtime conditions, do setup/teardown, etc. | |
|
104 | ||
|
105 | * The new ipcluster now has a fully working ssh mode that should work on | |
|
106 | Linux, Unix and OS X. Thanks to Vishal Vatsa for implementing this! | |
|
107 | ||
|
108 | * The wonderful TextMate editor can now be used with %edit on OS X. Thanks | |
|
109 | to Matt Foster for this patch. | |
|
110 | ||
|
111 | * The documentation regarding parallel uses of IPython, including MPI and PBS, | |
|
112 | has been significantly updated and improved. | |
|
113 | ||
|
114 | * The developer guidelines in the documentation have been updated to explain | |
|
115 | our workflow using :command:`bzr` and Launchpad. | |
|
116 | ||
|
117 | * Fully refactored :command:`ipcluster` command line program for starting | |
|
118 | IPython clusters. This new version is a complete rewrite and 1) is fully | |
|
119 | cross platform (we now use Twisted's process management), 2) has much | |
|
120 | improved performance, 3) uses subcommands for different types of clusters, 4) | |
|
121 | uses argparse for parsing command line options, 5) has better support for | |
|
122 | starting clusters using :command:`mpirun`, 6) has experimental support for | |
|
123 | starting engines using PBS. It can also reuse FURL files, by appropriately | |
|
124 | passing options to its subcommands. However, this new version of ipcluster | |
|
125 | should be considered a technology preview. We plan on changing the API in | |
|
126 | significant ways before it is final. | |
|
127 | ||
|
128 | * Full description of the security model added to the docs. | |
|
129 | ||
|
130 | * cd completer: show bookmarks if no other completions are available. | |
|
131 | ||
|
132 | * sh profile: easy way to give 'title' to prompt: assign to variable | |
|
133 | '_prompt_title'. It looks like this:: | |
|
134 | ||
|
135 | [~]|1> _prompt_title = 'sudo!' | |
|
136 | sudo![~]|2> | |
|
137 | ||
|
138 | * %edit: If you do '%edit pasted_block', pasted_block variable gets updated | |
|
139 | with new data (so repeated editing makes sense) | |
|
140 | ||
|
141 | .. _Numpy Documentation Standard: http://projects.scipy.org/numpy/wiki/CodingStyleGuidelines#docstring-standard | |
|
142 | ||
|
143 | .. _argparse: http://code.google.com/p/argparse/ | |
|
144 | ||
|
145 | Bug fixes | |
|
146 | --------- | |
|
147 | ||
|
148 | * Fix #368719, removed top-level debian/ directory to make the job of Debian | |
|
149 | packagers easier. | |
|
150 | ||
|
151 | * Fix #291143 by including man pages contributed by Stephan Peijnik from the | |
|
152 | Debian project. | |
|
153 | ||
|
154 | * Fix #358202, effectively a race condition, by properly synchronizing file | |
|
155 | creation at cluster startup time. | |
|
156 | ||
|
157 | * `%timeit` now handles correctly functions that take a long time to execute | |
|
158 | even the first time, by not repeating them. | |
|
159 | ||
|
160 | * Fix #239054, releasing of references after exiting. | |
|
161 | ||
|
162 | * Fix #341726, thanks to Alexander Clausen. | |
|
163 | ||
|
164 | * Fix #269966. This long-standing and very difficult bug (which is actually a | |
|
165 | problem in Python itself) meant long-running sessions would inevitably grow | |
|
166 | in memory size, often with catastrophic consequences if users had large | |
|
167 | objects in their scripts. Now, using `%run` repeatedly should not cause any | |
|
168 | memory leaks. Special thanks to John Hunter and Sameer D'Costa for their | |
|
169 | help with this bug. | |
|
170 | ||
|
171 | * Fix #295371, bug in `%history`. | |
|
172 | ||
|
173 | * Improved support for py2exe. | |
|
174 | ||
|
175 | * Fix #270856: IPython hangs with PyGTK | |
|
176 | ||
|
177 | * Fix #270998: A magic with no docstring breaks the '%magic magic' | |
|
178 | ||
|
179 | * fix #271684: -c startup commands screw up raw vs. native history | |
|
180 | ||
|
181 | * Numerous bugs on Windows with the new ipcluster have been fixed. | |
|
182 | ||
|
183 | * The ipengine and ipcontroller scripts now handle missing furl files | |
|
184 | more gracefully by giving better error messages. | |
|
185 | ||
|
186 | * %rehashx: Aliases no longer contain dots. python3.0 binary | |
|
187 | will create alias python30. Fixes: | |
|
188 | #259716 "commands with dots in them don't work" | |
|
189 | ||
|
190 | * %cpaste: %cpaste -r repeats the last pasted block. | |
|
191 | The block is assigned to pasted_block even if code | |
|
192 | raises exception. | |
|
193 | ||
|
194 | * Bug #274067 'The code in get_home_dir is broken for py2exe' was | |
|
195 | fixed. | |
|
196 | ||
|
197 | * Many other small bug fixes not listed here by number (see the bzr log for | |
|
198 | more info). | |
|
199 | ||
|
200 | Backwards incompatible changes | |
|
201 | ------------------------------ | |
|
202 | ||
|
203 | * `ipykit` and related files were unmaintained and have been removed. | |
|
204 | ||
|
205 | * The :func:`IPython.genutils.doctest_reload` does not actually call | |
|
206 | `reload(doctest)` anymore, as this was causing many problems with the test | |
|
207 | suite. It still resets `doctest.master` to None. | |
|
208 | ||
|
209 | * While we have not deliberately broken Python 2.4 compatibility, only minor | |
|
210 | testing was done with Python 2.4, while 2.5 and 2.6 were fully tested. But | |
|
211 | if you encounter problems with 2.4, please do report them as bugs. | |
|
212 | ||
|
213 | * The :command:`ipcluster` now requires a mode argument; for example to start a | |
|
214 | cluster on the local machine with 4 engines, you must now type:: | |
|
215 | ||
|
216 | $ ipcluster local -n 4 | |
|
217 | ||
|
218 | * The controller now has a ``-r`` flag that needs to be used if you want to | |
|
219 | reuse existing furl files. Otherwise they are deleted (the default). | |
|
220 | ||
|
221 | * Remove ipy_leo.py. You can use :command:`easy_install ipython-extension` to | |
|
222 | get it. (done to decouple it from ipython release cycle) | |
|
223 |
@@ -0,0 +1,34 b'' | |||
|
1 | ======================================== | |
|
2 | 0.8 series | |
|
3 | ======================================== | |
|
4 | ||
|
5 | Release 0.8.4 | |
|
6 | ============= | |
|
7 | ||
|
8 | This was a quick release to fix an unfortunate bug that slipped into the 0.8.3 | |
|
9 | release. The ``--twisted`` option was disabled, as it turned out to be broken | |
|
10 | across several platforms. | |
|
11 | ||
|
12 | ||
|
13 | Release 0.8.3 | |
|
14 | ============= | |
|
15 | ||
|
16 | * pydb is now disabled by default (due to %run -d problems). You can enable | |
|
17 | it by passing -pydb command line argument to IPython. Note that setting | |
|
18 | it in config file won't work. | |
|
19 | ||
|
20 | ||
|
21 | Release 0.8.2 | |
|
22 | ============= | |
|
23 | ||
|
24 | * %pushd/%popd behave differently; now "pushd /foo" pushes CURRENT directory | |
|
25 | and jumps to /foo. The current behaviour is closer to the documented | |
|
26 | behaviour, and should not trip anyone. | |
|
27 | ||
|
28 | ||
|
29 | Older releases | |
|
30 | ============== | |
|
31 | ||
|
32 | Changes in earlier releases of IPython are described in the older file | |
|
33 | ``ChangeLog``. Please refer to this document for details. | |
|
34 |
@@ -0,0 +1,283 b'' | |||
|
1 | ======================================== | |
|
2 | 0.9 series | |
|
3 | ======================================== | |
|
4 | ||
|
5 | Release 0.9.1 | |
|
6 | ============= | |
|
7 | ||
|
8 | This release was quickly made to restore compatibility with Python 2.4, which | |
|
9 | version 0.9 accidentally broke. No new features were introduced, other than | |
|
10 | some additional testing support for internal use. | |
|
11 | ||
|
12 | ||
|
13 | Release 0.9 | |
|
14 | =========== | |
|
15 | ||
|
16 | New features | |
|
17 | ------------ | |
|
18 | ||
|
19 | * All furl files and security certificates are now put in a read-only | |
|
20 | directory named ~./ipython/security. | |
|
21 | ||
|
22 | * A single function :func:`get_ipython_dir`, in :mod:`IPython.genutils` that | |
|
23 | determines the user's IPython directory in a robust manner. | |
|
24 | ||
|
25 | * Laurent's WX application has been given a top-level script called | |
|
26 | ipython-wx, and it has received numerous fixes. We expect this code to be | |
|
27 | architecturally better integrated with Gael's WX 'ipython widget' over the | |
|
28 | next few releases. | |
|
29 | ||
|
30 | * The Editor synchronization work by Vivian De Smedt has been merged in. This | |
|
31 | code adds a number of new editor hooks to synchronize with editors under | |
|
32 | Windows. | |
|
33 | ||
|
34 | * A new, still experimental but highly functional, WX shell by Gael Varoquaux. | |
|
35 | This work was sponsored by Enthought, and while it's still very new, it is | |
|
36 | based on a more cleanly organized arhictecture of the various IPython | |
|
37 | components. We will continue to develop this over the next few releases as a | |
|
38 | model for GUI components that use IPython. | |
|
39 | ||
|
40 | * Another GUI frontend, Cocoa based (Cocoa is the OSX native GUI framework), | |
|
41 | authored by Barry Wark. Currently the WX and the Cocoa ones have slightly | |
|
42 | different internal organizations, but the whole team is working on finding | |
|
43 | what the right abstraction points are for a unified codebase. | |
|
44 | ||
|
45 | * As part of the frontend work, Barry Wark also implemented an experimental | |
|
46 | event notification system that various ipython components can use. In the | |
|
47 | next release the implications and use patterns of this system regarding the | |
|
48 | various GUI options will be worked out. | |
|
49 | ||
|
50 | * IPython finally has a full test system, that can test docstrings with | |
|
51 | IPython-specific functionality. There are still a few pieces missing for it | |
|
52 | to be widely accessible to all users (so they can run the test suite at any | |
|
53 | time and report problems), but it now works for the developers. We are | |
|
54 | working hard on continuing to improve it, as this was probably IPython's | |
|
55 | major Achilles heel (the lack of proper test coverage made it effectively | |
|
56 | impossible to do large-scale refactoring). The full test suite can now | |
|
57 | be run using the :command:`iptest` command line program. | |
|
58 | ||
|
59 | * The notion of a task has been completely reworked. An `ITask` interface has | |
|
60 | been created. This interface defines the methods that tasks need to | |
|
61 | implement. These methods are now responsible for things like submitting | |
|
62 | tasks and processing results. There are two basic task types: | |
|
63 | :class:`IPython.kernel.task.StringTask` (this is the old `Task` object, but | |
|
64 | renamed) and the new :class:`IPython.kernel.task.MapTask`, which is based on | |
|
65 | a function. | |
|
66 | ||
|
67 | * A new interface, :class:`IPython.kernel.mapper.IMapper` has been defined to | |
|
68 | standardize the idea of a `map` method. This interface has a single `map` | |
|
69 | method that has the same syntax as the built-in `map`. We have also defined | |
|
70 | a `mapper` factory interface that creates objects that implement | |
|
71 | :class:`IPython.kernel.mapper.IMapper` for different controllers. Both the | |
|
72 | multiengine and task controller now have mapping capabilties. | |
|
73 | ||
|
74 | * The parallel function capabilities have been reworks. The major changes are | |
|
75 | that i) there is now an `@parallel` magic that creates parallel functions, | |
|
76 | ii) the syntax for mulitple variable follows that of `map`, iii) both the | |
|
77 | multiengine and task controller now have a parallel function implementation. | |
|
78 | ||
|
79 | * All of the parallel computing capabilities from `ipython1-dev` have been | |
|
80 | merged into IPython proper. This resulted in the following new subpackages: | |
|
81 | :mod:`IPython.kernel`, :mod:`IPython.kernel.core`, :mod:`IPython.config`, | |
|
82 | :mod:`IPython.tools` and :mod:`IPython.testing`. | |
|
83 | ||
|
84 | * As part of merging in the `ipython1-dev` stuff, the `setup.py` script and | |
|
85 | friends have been completely refactored. Now we are checking for | |
|
86 | dependencies using the approach that matplotlib uses. | |
|
87 | ||
|
88 | * The documentation has been completely reorganized to accept the | |
|
89 | documentation from `ipython1-dev`. | |
|
90 | ||
|
91 | * We have switched to using Foolscap for all of our network protocols in | |
|
92 | :mod:`IPython.kernel`. This gives us secure connections that are both | |
|
93 | encrypted and authenticated. | |
|
94 | ||
|
95 | * We have a brand new `COPYING.txt` files that describes the IPython license | |
|
96 | and copyright. The biggest change is that we are putting "The IPython | |
|
97 | Development Team" as the copyright holder. We give more details about | |
|
98 | exactly what this means in this file. All developer should read this and use | |
|
99 | the new banner in all IPython source code files. | |
|
100 | ||
|
101 | * sh profile: ./foo runs foo as system command, no need to do !./foo anymore | |
|
102 | ||
|
103 | * String lists now support ``sort(field, nums = True)`` method (to easily sort | |
|
104 | system command output). Try it with ``a = !ls -l ; a.sort(1, nums=1)``. | |
|
105 | ||
|
106 | * '%cpaste foo' now assigns the pasted block as string list, instead of string | |
|
107 | ||
|
108 | * The ipcluster script now run by default with no security. This is done | |
|
109 | because the main usage of the script is for starting things on localhost. | |
|
110 | Eventually when ipcluster is able to start things on other hosts, we will put | |
|
111 | security back. | |
|
112 | ||
|
113 | * 'cd --foo' searches directory history for string foo, and jumps to that dir. | |
|
114 | Last part of dir name is checked first. If no matches for that are found, | |
|
115 | look at the whole path. | |
|
116 | ||
|
117 | ||
|
118 | Bug fixes | |
|
119 | --------- | |
|
120 | ||
|
121 | * The Windows installer has been fixed. Now all IPython scripts have ``.bat`` | |
|
122 | versions created. Also, the Start Menu shortcuts have been updated. | |
|
123 | ||
|
124 | * The colors escapes in the multiengine client are now turned off on win32 as | |
|
125 | they don't print correctly. | |
|
126 | ||
|
127 | * The :mod:`IPython.kernel.scripts.ipengine` script was exec'ing | |
|
128 | mpi_import_statement incorrectly, which was leading the engine to crash when | |
|
129 | mpi was enabled. | |
|
130 | ||
|
131 | * A few subpackages had missing ``__init__.py`` files. | |
|
132 | ||
|
133 | * The documentation is only created if Sphinx is found. Previously, the | |
|
134 | ``setup.py`` script would fail if it was missing. | |
|
135 | ||
|
136 | * Greedy ``cd`` completion has been disabled again (it was enabled in 0.8.4) as | |
|
137 | it caused problems on certain platforms. | |
|
138 | ||
|
139 | ||
|
140 | Backwards incompatible changes | |
|
141 | ------------------------------ | |
|
142 | ||
|
143 | * The ``clusterfile`` options of the :command:`ipcluster` command has been | |
|
144 | removed as it was not working and it will be replaced soon by something much | |
|
145 | more robust. | |
|
146 | ||
|
147 | * The :mod:`IPython.kernel` configuration now properly find the user's | |
|
148 | IPython directory. | |
|
149 | ||
|
150 | * In ipapi, the :func:`make_user_ns` function has been replaced with | |
|
151 | :func:`make_user_namespaces`, to support dict subclasses in namespace | |
|
152 | creation. | |
|
153 | ||
|
154 | * :class:`IPython.kernel.client.Task` has been renamed | |
|
155 | :class:`IPython.kernel.client.StringTask` to make way for new task types. | |
|
156 | ||
|
157 | * The keyword argument `style` has been renamed `dist` in `scatter`, `gather` | |
|
158 | and `map`. | |
|
159 | ||
|
160 | * Renamed the values that the rename `dist` keyword argument can have from | |
|
161 | `'basic'` to `'b'`. | |
|
162 | ||
|
163 | * IPython has a larger set of dependencies if you want all of its capabilities. | |
|
164 | See the `setup.py` script for details. | |
|
165 | ||
|
166 | * The constructors for :class:`IPython.kernel.client.MultiEngineClient` and | |
|
167 | :class:`IPython.kernel.client.TaskClient` no longer take the (ip,port) tuple. | |
|
168 | Instead they take the filename of a file that contains the FURL for that | |
|
169 | client. If the FURL file is in your IPYTHONDIR, it will be found automatically | |
|
170 | and the constructor can be left empty. | |
|
171 | ||
|
172 | * The asynchronous clients in :mod:`IPython.kernel.asyncclient` are now created | |
|
173 | using the factory functions :func:`get_multiengine_client` and | |
|
174 | :func:`get_task_client`. These return a `Deferred` to the actual client. | |
|
175 | ||
|
176 | * The command line options to `ipcontroller` and `ipengine` have changed to | |
|
177 | reflect the new Foolscap network protocol and the FURL files. Please see the | |
|
178 | help for these scripts for details. | |
|
179 | ||
|
180 | * The configuration files for the kernel have changed because of the Foolscap | |
|
181 | stuff. If you were using custom config files before, you should delete them | |
|
182 | and regenerate new ones. | |
|
183 | ||
|
184 | Changes merged in from IPython1 | |
|
185 | ------------------------------- | |
|
186 | ||
|
187 | New features | |
|
188 | ............ | |
|
189 | ||
|
190 | * Much improved ``setup.py`` and ``setupegg.py`` scripts. Because Twisted and | |
|
191 | zope.interface are now easy installable, we can declare them as dependencies | |
|
192 | in our setupegg.py script. | |
|
193 | ||
|
194 | * IPython is now compatible with Twisted 2.5.0 and 8.x. | |
|
195 | ||
|
196 | * Added a new example of how to use :mod:`ipython1.kernel.asynclient`. | |
|
197 | ||
|
198 | * Initial draft of a process daemon in :mod:`ipython1.daemon`. This has not | |
|
199 | been merged into IPython and is still in `ipython1-dev`. | |
|
200 | ||
|
201 | * The ``TaskController`` now has methods for getting the queue status. | |
|
202 | ||
|
203 | * The ``TaskResult`` objects not have information about how long the task | |
|
204 | took to run. | |
|
205 | ||
|
206 | * We are attaching additional attributes to exceptions ``(_ipython_*)`` that | |
|
207 | we use to carry additional info around. | |
|
208 | ||
|
209 | * New top-level module :mod:`asyncclient` that has asynchronous versions (that | |
|
210 | return deferreds) of the client classes. This is designed to users who want | |
|
211 | to run their own Twisted reactor. | |
|
212 | ||
|
213 | * All the clients in :mod:`client` are now based on Twisted. This is done by | |
|
214 | running the Twisted reactor in a separate thread and using the | |
|
215 | :func:`blockingCallFromThread` function that is in recent versions of Twisted. | |
|
216 | ||
|
217 | * Functions can now be pushed/pulled to/from engines using | |
|
218 | :meth:`MultiEngineClient.push_function` and | |
|
219 | :meth:`MultiEngineClient.pull_function`. | |
|
220 | ||
|
221 | * Gather/scatter are now implemented in the client to reduce the work load | |
|
222 | of the controller and improve performance. | |
|
223 | ||
|
224 | * Complete rewrite of the IPython docuementation. All of the documentation | |
|
225 | from the IPython website has been moved into docs/source as restructured | |
|
226 | text documents. PDF and HTML documentation are being generated using | |
|
227 | Sphinx. | |
|
228 | ||
|
229 | * New developer oriented documentation: development guidelines and roadmap. | |
|
230 | ||
|
231 | * Traditional ``ChangeLog`` has been changed to a more useful ``changes.txt`` | |
|
232 | file that is organized by release and is meant to provide something more | |
|
233 | relevant for users. | |
|
234 | ||
|
235 | Bug fixes | |
|
236 | ......... | |
|
237 | ||
|
238 | * Created a proper ``MANIFEST.in`` file to create source distributions. | |
|
239 | ||
|
240 | * Fixed a bug in the ``MultiEngine`` interface. Previously, multi-engine | |
|
241 | actions were being collected with a :class:`DeferredList` with | |
|
242 | ``fireononeerrback=1``. This meant that methods were returning | |
|
243 | before all engines had given their results. This was causing extremely odd | |
|
244 | bugs in certain cases. To fix this problem, we have 1) set | |
|
245 | ``fireononeerrback=0`` to make sure all results (or exceptions) are in | |
|
246 | before returning and 2) introduced a :exc:`CompositeError` exception | |
|
247 | that wraps all of the engine exceptions. This is a huge change as it means | |
|
248 | that users will have to catch :exc:`CompositeError` rather than the actual | |
|
249 | exception. | |
|
250 | ||
|
251 | Backwards incompatible changes | |
|
252 | .............................. | |
|
253 | ||
|
254 | * All names have been renamed to conform to the lowercase_with_underscore | |
|
255 | convention. This will require users to change references to all names like | |
|
256 | ``queueStatus`` to ``queue_status``. | |
|
257 | ||
|
258 | * Previously, methods like :meth:`MultiEngineClient.push` and | |
|
259 | :meth:`MultiEngineClient.push` used ``*args`` and ``**kwargs``. This was | |
|
260 | becoming a problem as we weren't able to introduce new keyword arguments into | |
|
261 | the API. Now these methods simple take a dict or sequence. This has also | |
|
262 | allowed us to get rid of the ``*All`` methods like :meth:`pushAll` and | |
|
263 | :meth:`pullAll`. These things are now handled with the ``targets`` keyword | |
|
264 | argument that defaults to ``'all'``. | |
|
265 | ||
|
266 | * The :attr:`MultiEngineClient.magicTargets` has been renamed to | |
|
267 | :attr:`MultiEngineClient.targets`. | |
|
268 | ||
|
269 | * All methods in the MultiEngine interface now accept the optional keyword | |
|
270 | argument ``block``. | |
|
271 | ||
|
272 | * Renamed :class:`RemoteController` to :class:`MultiEngineClient` and | |
|
273 | :class:`TaskController` to :class:`TaskClient`. | |
|
274 | ||
|
275 | * Renamed the top-level module from :mod:`api` to :mod:`client`. | |
|
276 | ||
|
277 | * Most methods in the multiengine interface now raise a :exc:`CompositeError` | |
|
278 | exception that wraps the user's exceptions, rather than just raising the raw | |
|
279 | user's exception. | |
|
280 | ||
|
281 | * Changed the ``setupNS`` and ``resultNames`` in the ``Task`` class to ``push`` | |
|
282 | and ``pull``. | |
|
283 |
@@ -1,36 +1,42 b'' | |||
|
1 | 1 | #!/usr/bin/env python |
|
2 | 2 | """Script to auto-generate our API docs. |
|
3 | 3 | """ |
|
4 | 4 | # stdlib imports |
|
5 | 5 | import os |
|
6 | 6 | import sys |
|
7 | 7 | |
|
8 | 8 | # local imports |
|
9 | 9 | sys.path.append(os.path.abspath('sphinxext')) |
|
10 | 10 | from apigen import ApiDocWriter |
|
11 | 11 | |
|
12 | 12 | #***************************************************************************** |
|
13 | 13 | if __name__ == '__main__': |
|
14 | 14 | pjoin = os.path.join |
|
15 | 15 | package = 'IPython' |
|
16 | 16 | outdir = pjoin('source','api','generated') |
|
17 | 17 | docwriter = ApiDocWriter(package,rst_extension='.txt') |
|
18 | # You have to escape the . here because . is a special char for regexps. | |
|
19 | # You must do make clean if you change this! | |
|
18 | 20 | docwriter.package_skip_patterns += [r'\.fixes$', |
|
19 |
r'\.external |
|
|
21 | r'\.external$', | |
|
20 | 22 | r'\.extensions', |
|
21 | r'\.kernel.config', | |
|
23 | r'\.kernel\.config', | |
|
22 | 24 | r'\.attic', |
|
23 | 25 | r'\.quarantine', |
|
24 | r'\.deathrow' | |
|
26 | r'\.deathrow', | |
|
27 | r'\.config\.default', | |
|
28 | r'\.config\.profile', | |
|
29 | r'\.frontend', | |
|
30 | r'\.gui' | |
|
25 | 31 | ] |
|
26 | docwriter.module_skip_patterns += [ r'\.core.fakemodule', | |
|
32 | docwriter.module_skip_patterns += [ r'\.core\.fakemodule', | |
|
27 | 33 | r'\.cocoa', |
|
28 | 34 | r'\.ipdoctest', |
|
29 | 35 | r'\.Gnuplot', |
|
30 | r'\.frontend.process.winprocess', | |
|
36 | r'\.frontend\.process\.winprocess', | |
|
31 | 37 | ] |
|
32 | 38 | docwriter.write_api_docs(outdir) |
|
33 | 39 | docwriter.write_index(outdir, 'gen', |
|
34 | 40 | relative_to = pjoin('source','api') |
|
35 | 41 | ) |
|
36 | 42 | print '%d files written' % len(docwriter.written_modules) |
@@ -1,207 +1,208 b'' | |||
|
1 | 1 | .. _credits: |
|
2 | 2 | |
|
3 | 3 | ======= |
|
4 | 4 | Credits |
|
5 | 5 | ======= |
|
6 | 6 | |
|
7 | 7 | IPython is led by Fernando Pérez. |
|
8 | 8 | |
|
9 | 9 | As of this writing, the following developers have joined the core team: |
|
10 | 10 | |
|
11 | 11 | * [Robert Kern] <rkern-AT-enthought.com>: co-mentored the 2005 |
|
12 | 12 | Google Summer of Code project to develop python interactive |
|
13 | 13 | notebooks (XML documents) and graphical interface. This project |
|
14 | 14 | was awarded to the students Tzanko Matev <tsanko-AT-gmail.com> and |
|
15 | 15 | Toni Alatalo <antont-AT-an.org>. |
|
16 | 16 | |
|
17 | 17 | * [Brian Granger] <ellisonbg-AT-gmail.com>: extending IPython to allow |
|
18 | 18 | support for interactive parallel computing. |
|
19 | 19 | |
|
20 | 20 | * [Benjamin (Min) Ragan-Kelley]: key work on IPython's parallel |
|
21 | 21 | computing infrastructure. |
|
22 | 22 | |
|
23 | 23 | * [Ville Vainio] <vivainio-AT-gmail.com>: Ville has made many improvements |
|
24 | 24 | to the core of IPython and was the maintainer of the main IPython |
|
25 | 25 | trunk from version 0.7.1 to 0.8.4. |
|
26 | 26 | |
|
27 | 27 | * [Gael Varoquaux] <gael.varoquaux-AT-normalesup.org>: work on the merged |
|
28 | 28 | architecture for the interpreter as of version 0.9, implementing a new WX GUI |
|
29 | 29 | based on this system. |
|
30 | 30 | |
|
31 | 31 | * [Barry Wark] <barrywark-AT-gmail.com>: implementing a new Cocoa GUI, as well |
|
32 | 32 | as work on the new interpreter architecture and Twisted support. |
|
33 | 33 | |
|
34 | 34 | * [Laurent Dufrechou] <laurent.dufrechou-AT-gmail.com>: development of the WX |
|
35 | 35 | GUI support. |
|
36 | 36 | |
|
37 | 37 | * [Jörgen Stenarson] <jorgen.stenarson-AT-bostream.nu>: maintainer of the |
|
38 | 38 | PyReadline project, necessary for IPython under windows. |
|
39 | 39 | |
|
40 | 40 | |
|
41 | 41 | The IPython project is also very grateful to: |
|
42 | 42 | |
|
43 | 43 | Bill Bumgarner <bbum-AT-friday.com>: for providing the DPyGetOpt module |
|
44 | 44 | which gives very powerful and convenient handling of command-line |
|
45 | 45 | options (light years ahead of what Python 2.1.1's getopt module does). |
|
46 | 46 | |
|
47 | 47 | Ka-Ping Yee <ping-AT-lfw.org>: for providing the Itpl module for |
|
48 | 48 | convenient and powerful string interpolation with a much nicer syntax |
|
49 | 49 | than formatting through the '%' operator. |
|
50 | 50 | |
|
51 | 51 | Arnd Baecker <baecker-AT-physik.tu-dresden.de>: for his many very useful |
|
52 | 52 | suggestions and comments, and lots of help with testing and |
|
53 | 53 | documentation checking. Many of IPython's newer features are a result of |
|
54 | 54 | discussions with him (bugs are still my fault, not his). |
|
55 | 55 | |
|
56 | 56 | Obviously Guido van Rossum and the whole Python development team, that |
|
57 | 57 | goes without saying. |
|
58 | 58 | |
|
59 | 59 | IPython's website is generously hosted at http://ipython.scipy.orgby |
|
60 | 60 | Enthought (http://www.enthought.com). I am very grateful to them and all |
|
61 | 61 | of the SciPy team for their contribution. |
|
62 | 62 | |
|
63 | 63 | Fernando would also like to thank Stephen Figgins <fig-AT-monitor.net>, |
|
64 | 64 | an O'Reilly Python editor. His Oct/11/2001 article about IPP and |
|
65 | 65 | LazyPython, was what got this project started. You can read it at: |
|
66 | 66 | http://www.onlamp.com/pub/a/python/2001/10/11/pythonnews.html. |
|
67 | 67 | |
|
68 | 68 | And last but not least, all the kind IPython users who have emailed new code, |
|
69 | 69 | bug reports, fixes, comments and ideas. A brief list follows, please let us |
|
70 | 70 | know if we have ommitted your name by accident: |
|
71 | 71 | |
|
72 | 72 | * Dan Milstein <danmil-AT-comcast.net>. A bold refactoring of the |
|
73 | 73 | core prefilter stuff in the IPython interpreter. |
|
74 | 74 | |
|
75 | 75 | * [Jack Moffit] <jack-AT-xiph.org> Bug fixes, including the infamous |
|
76 | 76 | color problem. This bug alone caused many lost hours and |
|
77 | 77 | frustration, many thanks to him for the fix. I've always been a |
|
78 | 78 | fan of Ogg & friends, now I have one more reason to like these folks. |
|
79 | 79 | Jack is also contributing with Debian packaging and many other |
|
80 | 80 | things. |
|
81 | 81 | |
|
82 | 82 | * [Alexander Schmolck] <a.schmolck-AT-gmx.net> Emacs work, bug |
|
83 | 83 | reports, bug fixes, ideas, lots more. The ipython.el mode for |
|
84 | 84 | (X)Emacs is Alex's code, providing full support for IPython under |
|
85 | 85 | (X)Emacs. |
|
86 | 86 | |
|
87 | 87 | * [Andrea Riciputi] <andrea.riciputi-AT-libero.it> Mac OSX |
|
88 | 88 | information, Fink package management. |
|
89 | 89 | |
|
90 | 90 | * [Gary Bishop] <gb-AT-cs.unc.edu> Bug reports, and patches to work |
|
91 | 91 | around the exception handling idiosyncracies of WxPython. Readline |
|
92 | 92 | and color support for Windows. |
|
93 | 93 | |
|
94 | 94 | * [Jeffrey Collins] <Jeff.Collins-AT-vexcel.com> Bug reports. Much |
|
95 | 95 | improved readline support, including fixes for Python 2.3. |
|
96 | 96 | |
|
97 | 97 | * [Dryice Liu] <dryice-AT-liu.com.cn> FreeBSD port. |
|
98 | 98 | |
|
99 | 99 | * [Mike Heeter] <korora-AT-SDF.LONESTAR.ORG> |
|
100 | 100 | |
|
101 | 101 | * [Christopher Hart] <hart-AT-caltech.edu> PDB integration. |
|
102 | 102 | |
|
103 | 103 | * [Milan Zamazal] <pdm-AT-zamazal.org> Emacs info. |
|
104 | 104 | |
|
105 | 105 | * [Philip Hisley] <compsys-AT-starpower.net> |
|
106 | 106 | |
|
107 | 107 | * [Holger Krekel] <pyth-AT-devel.trillke.net> Tab completion, lots |
|
108 | 108 | more. |
|
109 | 109 | |
|
110 | 110 | * [Robin Siebler] <robinsiebler-AT-starband.net> |
|
111 | 111 | |
|
112 | 112 | * [Ralf Ahlbrink] <ralf_ahlbrink-AT-web.de> |
|
113 | 113 | |
|
114 | 114 | * [Thorsten Kampe] <thorsten-AT-thorstenkampe.de> |
|
115 | 115 | |
|
116 | 116 | * [Fredrik Kant] <fredrik.kant-AT-front.com> Windows setup. |
|
117 | 117 | |
|
118 | 118 | * [Syver Enstad] <syver-en-AT-online.no> Windows setup. |
|
119 | 119 | |
|
120 | 120 | * [Richard] <rxe-AT-renre-europe.com> Global embedding. |
|
121 | 121 | |
|
122 | 122 | * [Hayden Callow] <h.callow-AT-elec.canterbury.ac.nz> Gnuplot.py 1.6 |
|
123 | 123 | compatibility. |
|
124 | 124 | |
|
125 | 125 | * [Leonardo Santagada] <retype-AT-terra.com.br> Fixes for Windows |
|
126 | 126 | installation. |
|
127 | 127 | |
|
128 | 128 | * [Christopher Armstrong] <radix-AT-twistedmatrix.com> Bugfixes. |
|
129 | 129 | |
|
130 | 130 | * [Francois Pinard] <pinard-AT-iro.umontreal.ca> Code and |
|
131 | 131 | documentation fixes. |
|
132 | 132 | |
|
133 | 133 | * [Cory Dodt] <cdodt-AT-fcoe.k12.ca.us> Bug reports and Windows |
|
134 | 134 | ideas. Patches for Windows installer. |
|
135 | 135 | |
|
136 | 136 | * [Olivier Aubert] <oaubert-AT-bat710.univ-lyon1.fr> New magics. |
|
137 | 137 | |
|
138 | 138 | * [King C. Shu] <kingshu-AT-myrealbox.com> Autoindent patch. |
|
139 | 139 | |
|
140 | 140 | * [Chris Drexler] <chris-AT-ac-drexler.de> Readline packages for |
|
141 | 141 | Win32/CygWin. |
|
142 | 142 | |
|
143 | 143 | * [Gustavo Cordova Avila] <gcordova-AT-sismex.com> EvalDict code for |
|
144 | 144 | nice, lightweight string interpolation. |
|
145 | 145 | |
|
146 | 146 | * [Kasper Souren] <Kasper.Souren-AT-ircam.fr> Bug reports, ideas. |
|
147 | 147 | |
|
148 | 148 | * [Gever Tulley] <gever-AT-helium.com> Code contributions. |
|
149 | 149 | |
|
150 | 150 | * [Ralf Schmitt] <ralf-AT-brainbot.com> Bug reports & fixes. |
|
151 | 151 | |
|
152 | 152 | * [Oliver Sander] <osander-AT-gmx.de> Bug reports. |
|
153 | 153 | |
|
154 | 154 | * [Rod Holland] <rhh-AT-structurelabs.com> Bug reports and fixes to |
|
155 | 155 | logging module. |
|
156 | 156 | |
|
157 | 157 | * [Daniel 'Dang' Griffith] <pythondev-dang-AT-lazytwinacres.net> |
|
158 | 158 | Fixes, enhancement suggestions for system shell use. |
|
159 | 159 | |
|
160 | 160 | * [Viktor Ransmayr] <viktor.ransmayr-AT-t-online.de> Tests and |
|
161 | 161 | reports on Windows installation issues. Contributed a true Windows |
|
162 | 162 | binary installer. |
|
163 | 163 | |
|
164 | 164 | * [Mike Salib] <msalib-AT-mit.edu> Help fixing a subtle bug related |
|
165 | 165 | to traceback printing. |
|
166 | 166 | |
|
167 | 167 | * [W.J. van der Laan] <gnufnork-AT-hetdigitalegat.nl> Bash-like |
|
168 | 168 | prompt specials. |
|
169 | 169 | |
|
170 | 170 | * [Antoon Pardon] <Antoon.Pardon-AT-rece.vub.ac.be> Critical fix for |
|
171 | 171 | the multithreaded IPython. |
|
172 | 172 | |
|
173 | 173 | * [John Hunter] <jdhunter-AT-nitace.bsd.uchicago.edu> Matplotlib |
|
174 | 174 | author, helped with all the development of support for matplotlib |
|
175 | 175 | in IPyhton, including making necessary changes to matplotlib itself. |
|
176 | 176 | |
|
177 | 177 | * [Matthew Arnison] <maffew-AT-cat.org.au> Bug reports, '%run -d' idea. |
|
178 | 178 | |
|
179 | 179 | * [Prabhu Ramachandran] <prabhu_r-AT-users.sourceforge.net> Help |
|
180 | 180 | with (X)Emacs support, threading patches, ideas... |
|
181 | 181 | |
|
182 | 182 | * [Norbert Tretkowski] <tretkowski-AT-inittab.de> help with Debian |
|
183 | 183 | packaging and distribution. |
|
184 | 184 | |
|
185 | 185 | * [George Sakkis] <gsakkis-AT-eden.rutgers.edu> New matcher for |
|
186 | 186 | tab-completing named arguments of user-defined functions. |
|
187 | 187 | |
|
188 | 188 | * [Jörgen Stenarson] <jorgen.stenarson-AT-bostream.nu> Wildcard |
|
189 | 189 | support implementation for searching namespaces. |
|
190 | 190 | |
|
191 | 191 | * [Vivian De Smedt] <vivian-AT-vdesmedt.com> Debugger enhancements, |
|
192 | 192 | so that when pdb is activated from within IPython, coloring, tab |
|
193 | 193 | completion and other features continue to work seamlessly. |
|
194 | 194 | |
|
195 | 195 | * [Scott Tsai] <scottt958-AT-yahoo.com.tw> Support for automatic |
|
196 | 196 | editor invocation on syntax errors (see |
|
197 | 197 | http://www.scipy.net/roundup/ipython/issue36). |
|
198 | 198 | |
|
199 | 199 | * [Alexander Belchenko] <bialix-AT-ukr.net> Improvements for win32 |
|
200 | 200 | paging system. |
|
201 | 201 | |
|
202 | 202 | * [Will Maier] <willmaier-AT-ml1.net> Official OpenBSD port. |
|
203 | 203 | |
|
204 | 204 | * [Ondrej Certik] <ondrej-AT-certik.cz>: set up the IPython docs to use the new |
|
205 | 205 | Sphinx system used by Python, Matplotlib and many more projects. |
|
206 | 206 | |
|
207 | 207 | * [Stefan van der Walt] <stefan-AT-sun.ac.za>: support for the new config system. |
|
208 |
|
1 | NO CONTENT: file renamed from docs/source/history.txt to docs/source/about/history.txt |
@@ -1,91 +1,92 b'' | |||
|
1 | 1 | .. _license: |
|
2 | 2 | |
|
3 | 3 | ===================== |
|
4 | 4 | License and Copyright |
|
5 | 5 | ===================== |
|
6 | 6 | |
|
7 | 7 | License |
|
8 | 8 | ======= |
|
9 | 9 | |
|
10 | 10 | IPython is licensed under the terms of the new or revised BSD license, as follows:: |
|
11 | 11 | |
|
12 | 12 | Copyright (c) 2008, IPython Development Team |
|
13 | 13 | |
|
14 | 14 | All rights reserved. |
|
15 | 15 | |
|
16 | 16 | Redistribution and use in source and binary forms, with or without |
|
17 | 17 | modification, are permitted provided that the following conditions are |
|
18 | 18 | met: |
|
19 | 19 | |
|
20 | 20 | Redistributions of source code must retain the above copyright notice, |
|
21 | 21 | this list of conditions and the following disclaimer. |
|
22 | 22 | |
|
23 | 23 | Redistributions in binary form must reproduce the above copyright notice, |
|
24 | 24 | this list of conditions and the following disclaimer in the documentation |
|
25 | 25 | and/or other materials provided with the distribution. |
|
26 | 26 | |
|
27 | 27 | Neither the name of the IPython Development Team nor the names of its |
|
28 | 28 | contributors may be used to endorse or promote products derived from this |
|
29 | 29 | software without specific prior written permission. |
|
30 | 30 | |
|
31 | 31 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS |
|
32 | 32 | IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, |
|
33 | 33 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR |
|
34 | 34 | PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR |
|
35 | 35 | CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, |
|
36 | 36 | EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, |
|
37 | 37 | PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR |
|
38 | 38 | PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF |
|
39 | 39 | LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING |
|
40 | 40 | NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS |
|
41 | 41 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
|
42 | 42 | |
|
43 | 43 | About the IPython Development Team |
|
44 | 44 | ================================== |
|
45 | 45 | |
|
46 | 46 | Fernando Perez began IPython in 2001 based on code from Janko Hauser |
|
47 | 47 | <jhauser@zscout.de> and Nathaniel Gray <n8gray@caltech.edu>. Fernando is still |
|
48 | 48 | the project lead. |
|
49 | 49 | |
|
50 | 50 | The IPython Development Team is the set of all contributors to the IPython |
|
51 | 51 | project. This includes all of the IPython subprojects. Here is a list of the |
|
52 | 52 | currently active contributors: |
|
53 | 53 | |
|
54 | 54 | * Matthieu Brucher |
|
55 | 55 | * Ondrej Certik |
|
56 | 56 | * Laurent Dufrechou |
|
57 | 57 | * Robert Kern |
|
58 | 58 | * Brian E. Granger |
|
59 | 59 | * Fernando Perez (project leader) |
|
60 | 60 | * Benjamin Ragan-Kelley |
|
61 | 61 | * Ville M. Vainio |
|
62 | 62 | * Gael Varoququx |
|
63 | 63 | * Stefan van der Walt |
|
64 | 64 | * Tech-X Corporation |
|
65 | 65 | * Barry Wark |
|
66 | 66 | |
|
67 | 67 | If your name is missing, please add it. |
|
68 | 68 | |
|
69 | 69 | Our Copyright Policy |
|
70 | 70 | ==================== |
|
71 | 71 | |
|
72 | 72 | IPython uses a shared copyright model. Each contributor maintains copyright |
|
73 | 73 | over their contributions to IPython. But, it is important to note that these |
|
74 | 74 | contributions are typically only changes to the repositories. Thus, the |
|
75 | 75 | IPython source code, in its entirety is not the copyright of any single person |
|
76 | 76 | or institution. Instead, it is the collective copyright of the entire IPython |
|
77 | 77 | Development Team. If individual contributors want to maintain a record of what |
|
78 | 78 | changes/contributions they have specific copyright on, they should indicate |
|
79 | 79 | their copyright in the commit message of the change, when they commit the |
|
80 | 80 | change to one of the IPython repositories. |
|
81 | 81 | |
|
82 | 82 | Miscellaneous |
|
83 | 83 | ============= |
|
84 | 84 | |
|
85 | 85 | Some files (DPyGetOpt.py, for example) may be licensed under different |
|
86 | 86 | conditions. Ultimately each file indicates clearly the conditions under which |
|
87 | 87 | its author/authors have decided to publish the code. |
|
88 | 88 | |
|
89 | 89 | Versions of IPython up to and including 0.6.3 were released under the GNU |
|
90 | 90 | Lesser General Public License (LGPL), available at |
|
91 | http://www.gnu.org/copyleft/lesser.html. No newline at end of file | |
|
91 | http://www.gnu.org/copyleft/lesser.html. | |
|
92 |
@@ -1,191 +1,192 b'' | |||
|
1 | 1 | # -*- coding: utf-8 -*- |
|
2 | 2 | # |
|
3 | 3 | # IPython documentation build configuration file. |
|
4 | 4 | |
|
5 | 5 | # NOTE: This file has been edited manually from the auto-generated one from |
|
6 | 6 | # sphinx. Do NOT delete and re-generate. If any changes from sphinx are |
|
7 | 7 | # needed, generate a scratch one and merge by hand any new fields needed. |
|
8 | 8 | |
|
9 | 9 | # |
|
10 | 10 | # This file is execfile()d with the current directory set to its containing dir. |
|
11 | 11 | # |
|
12 | 12 | # The contents of this file are pickled, so don't put values in the namespace |
|
13 | 13 | # that aren't pickleable (module imports are okay, they're removed automatically). |
|
14 | 14 | # |
|
15 | 15 | # All configuration values have a default value; values that are commented out |
|
16 | 16 | # serve to show the default value. |
|
17 | 17 | |
|
18 | 18 | import sys, os |
|
19 | 19 | |
|
20 | 20 | # If your extensions are in another directory, add it here. If the directory |
|
21 | 21 | # is relative to the documentation root, use os.path.abspath to make it |
|
22 | 22 | # absolute, like shown here. |
|
23 | 23 | sys.path.append(os.path.abspath('../sphinxext')) |
|
24 | 24 | |
|
25 | 25 | # Import support for ipython console session syntax highlighting (lives |
|
26 | 26 | # in the sphinxext directory defined above) |
|
27 | 27 | import ipython_console_highlighting |
|
28 | 28 | |
|
29 | 29 | # We load the ipython release info into a dict by explicit execution |
|
30 | 30 | iprelease = {} |
|
31 | 31 | execfile('../../IPython/core/release.py',iprelease) |
|
32 | 32 | |
|
33 | 33 | # General configuration |
|
34 | 34 | # --------------------- |
|
35 | 35 | |
|
36 | 36 | # Add any Sphinx extension module names here, as strings. They can be extensions |
|
37 | 37 | # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. |
|
38 | extensions = ['sphinx.ext.autodoc', | |
|
39 | 'sphinx.ext.doctest', | |
|
40 | ||
|
41 | 'only_directives', | |
|
42 | 'inheritance_diagram', | |
|
43 | 'ipython_console_highlighting', | |
|
44 | # 'plot_directive', # disabled for now, needs matplotlib | |
|
45 | 'numpydoc', # to preprocess docstrings | |
|
46 | ] | |
|
38 | extensions = [ | |
|
39 | # 'matplotlib.sphinxext.mathmpl', | |
|
40 | 'matplotlib.sphinxext.only_directives', | |
|
41 | # 'matplotlib.sphinxext.plot_directive', | |
|
42 | 'sphinx.ext.autodoc', | |
|
43 | 'sphinx.ext.doctest', | |
|
44 | 'inheritance_diagram', | |
|
45 | 'ipython_console_highlighting', | |
|
46 | 'numpydoc', # to preprocess docstrings | |
|
47 | ] | |
|
47 | 48 | |
|
48 | 49 | # Add any paths that contain templates here, relative to this directory. |
|
49 | 50 | templates_path = ['_templates'] |
|
50 | 51 | |
|
51 | 52 | # The suffix of source filenames. |
|
52 | 53 | source_suffix = '.txt' |
|
53 | 54 | |
|
54 | 55 | # The master toctree document. |
|
55 | 56 | master_doc = 'index' |
|
56 | 57 | |
|
57 | 58 | # General substitutions. |
|
58 | 59 | project = 'IPython' |
|
59 | 60 | copyright = '2008, The IPython Development Team' |
|
60 | 61 | |
|
61 | 62 | # The default replacements for |version| and |release|, also used in various |
|
62 | 63 | # other places throughout the built documents. |
|
63 | 64 | # |
|
64 | 65 | # The full version, including alpha/beta/rc tags. |
|
65 | 66 | release = iprelease['version'] |
|
66 | 67 | # The short X.Y version. |
|
67 | 68 | version = '.'.join(release.split('.',2)[:2]) |
|
68 | 69 | |
|
69 | 70 | |
|
70 | 71 | # There are two options for replacing |today|: either, you set today to some |
|
71 | 72 | # non-false value, then it is used: |
|
72 | 73 | #today = '' |
|
73 | 74 | # Else, today_fmt is used as the format for a strftime call. |
|
74 | 75 | today_fmt = '%B %d, %Y' |
|
75 | 76 | |
|
76 | 77 | # List of documents that shouldn't be included in the build. |
|
77 | 78 | #unused_docs = [] |
|
78 | 79 | |
|
79 | 80 | # List of directories, relative to source directories, that shouldn't be searched |
|
80 | 81 | # for source files. |
|
81 | 82 | exclude_dirs = ['attic'] |
|
82 | 83 | |
|
83 | 84 | # If true, '()' will be appended to :func: etc. cross-reference text. |
|
84 | 85 | #add_function_parentheses = True |
|
85 | 86 | |
|
86 | 87 | # If true, the current module name will be prepended to all description |
|
87 | 88 | # unit titles (such as .. function::). |
|
88 | 89 | #add_module_names = True |
|
89 | 90 | |
|
90 | 91 | # If true, sectionauthor and moduleauthor directives will be shown in the |
|
91 | 92 | # output. They are ignored by default. |
|
92 | 93 | #show_authors = False |
|
93 | 94 | |
|
94 | 95 | # The name of the Pygments (syntax highlighting) style to use. |
|
95 | 96 | pygments_style = 'sphinx' |
|
96 | 97 | |
|
97 | 98 | |
|
98 | 99 | # Options for HTML output |
|
99 | 100 | # ----------------------- |
|
100 | 101 | |
|
101 | 102 | # The style sheet to use for HTML and HTML Help pages. A file of that name |
|
102 | 103 | # must exist either in Sphinx' static/ path, or in one of the custom paths |
|
103 | 104 | # given in html_static_path. |
|
104 | 105 | html_style = 'default.css' |
|
105 | 106 | |
|
106 | 107 | # The name for this set of Sphinx documents. If None, it defaults to |
|
107 | 108 | # "<project> v<release> documentation". |
|
108 | 109 | #html_title = None |
|
109 | 110 | |
|
110 | 111 | # The name of an image file (within the static path) to place at the top of |
|
111 | 112 | # the sidebar. |
|
112 | 113 | #html_logo = None |
|
113 | 114 | |
|
114 | 115 | # Add any paths that contain custom static files (such as style sheets) here, |
|
115 | 116 | # relative to this directory. They are copied after the builtin static files, |
|
116 | 117 | # so a file named "default.css" will overwrite the builtin "default.css". |
|
117 | 118 | html_static_path = ['_static'] |
|
118 | 119 | |
|
119 | 120 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, |
|
120 | 121 | # using the given strftime format. |
|
121 | 122 | html_last_updated_fmt = '%b %d, %Y' |
|
122 | 123 | |
|
123 | 124 | # If true, SmartyPants will be used to convert quotes and dashes to |
|
124 | 125 | # typographically correct entities. |
|
125 | 126 | #html_use_smartypants = True |
|
126 | 127 | |
|
127 | 128 | # Custom sidebar templates, maps document names to template names. |
|
128 | 129 | #html_sidebars = {} |
|
129 | 130 | |
|
130 | 131 | # Additional templates that should be rendered to pages, maps page names to |
|
131 | 132 | # template names. |
|
132 | 133 | #html_additional_pages = {} |
|
133 | 134 | |
|
134 | 135 | # If false, no module index is generated. |
|
135 | 136 | #html_use_modindex = True |
|
136 | 137 | |
|
137 | 138 | # If true, the reST sources are included in the HTML build as _sources/<name>. |
|
138 | 139 | #html_copy_source = True |
|
139 | 140 | |
|
140 | 141 | # If true, an OpenSearch description file will be output, and all pages will |
|
141 | 142 | # contain a <link> tag referring to it. The value of this option must be the |
|
142 | 143 | # base URL from which the finished HTML is served. |
|
143 | 144 | #html_use_opensearch = '' |
|
144 | 145 | |
|
145 | 146 | # If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml"). |
|
146 | 147 | #html_file_suffix = '' |
|
147 | 148 | |
|
148 | 149 | # Output file base name for HTML help builder. |
|
149 | 150 | htmlhelp_basename = 'ipythondoc' |
|
150 | 151 | |
|
151 | 152 | |
|
152 | 153 | # Options for LaTeX output |
|
153 | 154 | # ------------------------ |
|
154 | 155 | |
|
155 | 156 | # The paper size ('letter' or 'a4'). |
|
156 | 157 | latex_paper_size = 'letter' |
|
157 | 158 | |
|
158 | 159 | # The font size ('10pt', '11pt' or '12pt'). |
|
159 | 160 | latex_font_size = '11pt' |
|
160 | 161 | |
|
161 | 162 | # Grouping the document tree into LaTeX files. List of tuples |
|
162 | 163 | # (source start file, target name, title, author, document class [howto/manual]). |
|
163 | 164 | |
|
164 | 165 | latex_documents = [ ('index', 'ipython.tex', 'IPython Documentation', |
|
165 | 166 | ur"""The IPython Development Team""", |
|
166 | 167 | 'manual'), |
|
167 | 168 | ] |
|
168 | 169 | |
|
169 | 170 | # The name of an image file (relative to this directory) to place at the top of |
|
170 | 171 | # the title page. |
|
171 | 172 | #latex_logo = None |
|
172 | 173 | |
|
173 | 174 | # For "manual" documents, if this is true, then toplevel headings are parts, |
|
174 | 175 | # not chapters. |
|
175 | 176 | #latex_use_parts = False |
|
176 | 177 | |
|
177 | 178 | # Additional stuff for the LaTeX preamble. |
|
178 | 179 | #latex_preamble = '' |
|
179 | 180 | |
|
180 | 181 | # Documents to append as an appendix to all manuals. |
|
181 | 182 | #latex_appendices = [] |
|
182 | 183 | |
|
183 | 184 | # If false, no module index is generated. |
|
184 | 185 | #latex_use_modindex = True |
|
185 | 186 | |
|
186 | 187 | |
|
187 | 188 | # Cleanup |
|
188 | 189 | # ------- |
|
189 | 190 | # delete release info to avoid pickling errors from sphinx |
|
190 | 191 | |
|
191 | 192 | del iprelease |
@@ -1,286 +1,287 b'' | |||
|
1 | 1 | .. _customization: |
|
2 | 2 | |
|
3 | 3 | ======================== |
|
4 | 4 | Customization of IPython |
|
5 | 5 | ======================== |
|
6 | 6 | |
|
7 | 7 | There are 2 ways to configure IPython - the old way of using ipythonrc |
|
8 | 8 | files (an INI-file like format), and the new way that involves editing |
|
9 | 9 | your ipy_user_conf.py. Both configuration systems work at the same |
|
10 | 10 | time, so you can set your options in both, but if you are hesitating |
|
11 | 11 | about which alternative to choose, we recommend the ipy_user_conf.py |
|
12 | 12 | approach, as it will give you more power and control in the long |
|
13 | 13 | run. However, there are few options such as pylab_import_all that can |
|
14 | 14 | only be specified in ipythonrc file or command line - the reason for |
|
15 | 15 | this is that they are needed before IPython has been started up, and |
|
16 | 16 | the IPApi object used in ipy_user_conf.py is not yet available at that |
|
17 | 17 | time. A hybrid approach of specifying a few options in ipythonrc and |
|
18 | 18 | doing the more advanced configuration in ipy_user_conf.py is also |
|
19 | 19 | possible. |
|
20 | 20 | |
|
21 | 21 | .. _ipythonrc: |
|
22 | 22 | |
|
23 | 23 | The ipythonrc approach |
|
24 | 24 | ====================== |
|
25 | 25 | |
|
26 | 26 | As we've already mentioned, IPython reads a configuration file which can |
|
27 | 27 | be specified at the command line (-rcfile) or which by default is |
|
28 | 28 | assumed to be called ipythonrc. Such a file is looked for in the current |
|
29 | 29 | directory where IPython is started and then in your IPYTHONDIR, which |
|
30 | 30 | allows you to have local configuration files for specific projects. In |
|
31 | 31 | this section we will call these types of configuration files simply |
|
32 | 32 | rcfiles (short for resource configuration file). |
|
33 | 33 | |
|
34 | 34 | The syntax of an rcfile is one of key-value pairs separated by |
|
35 | 35 | whitespace, one per line. Lines beginning with a # are ignored as |
|
36 | 36 | comments, but comments can not be put on lines with data (the parser is |
|
37 | 37 | fairly primitive). Note that these are not python files, and this is |
|
38 | 38 | deliberate, because it allows us to do some things which would be quite |
|
39 | 39 | tricky to implement if they were normal python files. |
|
40 | 40 | |
|
41 | 41 | First, an rcfile can contain permanent default values for almost all command |
|
42 | 42 | line options (except things like -help or -Version). :ref:`This section |
|
43 | 43 | <command_line_options>` contains a description of all command-line |
|
44 | 44 | options. However, values you explicitly specify at the command line override |
|
45 | 45 | the values defined in the rcfile. |
|
46 | 46 | |
|
47 | 47 | Besides command line option values, the rcfile can specify values for |
|
48 | 48 | certain extra special options which are not available at the command |
|
49 | 49 | line. These options are briefly described below. |
|
50 | 50 | |
|
51 | 51 | Each of these options may appear as many times as you need it in the file. |
|
52 | 52 | |
|
53 | 53 | * include <file1> <file2> ...: you can name other rcfiles you want |
|
54 | 54 | to recursively load up to 15 levels (don't use the <> brackets in |
|
55 | 55 | your names!). This feature allows you to define a 'base' rcfile |
|
56 | 56 | with general options and special-purpose files which can be loaded |
|
57 | 57 | only when needed with particular configuration options. To make |
|
58 | 58 | this more convenient, IPython accepts the -profile <name> option |
|
59 | 59 | (abbreviates to -p <name>) which tells it to look for an rcfile |
|
60 | 60 | named ipythonrc-<name>. |
|
61 | 61 | * import_mod <mod1> <mod2> ...: import modules with 'import |
|
62 | 62 | <mod1>,<mod2>,...' |
|
63 | 63 | * import_some <mod> <f1> <f2> ...: import functions with 'from |
|
64 | 64 | <mod> import <f1>,<f2>,...' |
|
65 | 65 | * import_all <mod1> <mod2> ...: for each module listed import |
|
66 | 66 | functions with ``from <mod> import *``. |
|
67 | 67 | * execute <python code>: give any single-line python code to be |
|
68 | 68 | executed. |
|
69 | 69 | * execfile <filename>: execute the python file given with an |
|
70 | 70 | 'execfile(filename)' command. Username expansion is performed on |
|
71 | 71 | the given names. So if you need any amount of extra fancy |
|
72 | 72 | customization that won't fit in any of the above 'canned' options, |
|
73 | 73 | you can just put it in a separate python file and execute it. |
|
74 | 74 | * alias <alias_def>: this is equivalent to calling |
|
75 | 75 | '%alias <alias_def>' at the IPython command line. This way, from |
|
76 | 76 | within IPython you can do common system tasks without having to |
|
77 | 77 | exit it or use the ! escape. IPython isn't meant to be a shell |
|
78 | 78 | replacement, but it is often very useful to be able to do things |
|
79 | 79 | with files while testing code. This gives you the flexibility to |
|
80 | 80 | have within IPython any aliases you may be used to under your |
|
81 | 81 | normal system shell. |
|
82 | 82 | |
|
83 | 83 | ipy_user_conf.py |
|
84 | 84 | ================ |
|
85 | 85 | |
|
86 | 86 | There should be a simple template ipy_user_conf.py file in your |
|
87 | 87 | ~/.ipython directory. It is a plain python module that is imported |
|
88 | 88 | during IPython startup, so you can do pretty much what you want there |
|
89 | 89 | - import modules, configure extensions, change options, define magic |
|
90 | 90 | commands, put variables and functions in the IPython namespace, |
|
91 | 91 | etc. You use the IPython extension api object, acquired by |
|
92 | 92 | IPython.ipapi.get() and documented in the "IPython extension API" |
|
93 | 93 | chapter, to interact with IPython. A sample ipy_user_conf.py is listed |
|
94 | 94 | below for reference:: |
|
95 | 95 | |
|
96 | 96 | # Most of your config files and extensions will probably start |
|
97 | 97 | # with this import |
|
98 | 98 | |
|
99 | 99 | import IPython.ipapi |
|
100 | 100 | ip = IPython.ipapi.get() |
|
101 | 101 | |
|
102 | 102 | # You probably want to uncomment this if you did %upgrade -nolegacy |
|
103 | 103 | # import ipy_defaults |
|
104 | 104 | |
|
105 | 105 | import os |
|
106 | 106 | |
|
107 | 107 | def main(): |
|
108 | 108 | |
|
109 | 109 | #ip.dbg.debugmode = True |
|
110 | 110 | ip.dbg.debug_stack() |
|
111 | 111 | |
|
112 | 112 | # uncomment if you want to get ipython -p sh behaviour |
|
113 | 113 | # without having to use command line switches |
|
114 | 114 | import ipy_profile_sh |
|
115 | 115 | import jobctrl |
|
116 | 116 | |
|
117 | 117 | # Configure your favourite editor? |
|
118 | 118 | # Good idea e.g. for %edit os.path.isfile |
|
119 | 119 | |
|
120 | 120 | #import ipy_editors |
|
121 | 121 | |
|
122 | 122 | # Choose one of these: |
|
123 | 123 | |
|
124 | 124 | #ipy_editors.scite() |
|
125 | 125 | #ipy_editors.scite('c:/opt/scite/scite.exe') |
|
126 | 126 | #ipy_editors.komodo() |
|
127 | 127 | #ipy_editors.idle() |
|
128 | 128 | # ... or many others, try 'ipy_editors??' after import to see them |
|
129 | 129 | |
|
130 | 130 | # Or roll your own: |
|
131 | 131 | #ipy_editors.install_editor("c:/opt/jed +$line $file") |
|
132 | 132 | |
|
133 | 133 | |
|
134 | 134 | o = ip.options |
|
135 | 135 | # An example on how to set options |
|
136 | 136 | #o.autocall = 1 |
|
137 | 137 | o.system_verbose = 0 |
|
138 | 138 | |
|
139 | 139 | #import_all("os sys") |
|
140 | 140 | #execf('~/_ipython/ns.py') |
|
141 | 141 | |
|
142 | 142 | |
|
143 | 143 | # -- prompt |
|
144 | 144 | # A different, more compact set of prompts from the default ones, that |
|
145 | 145 | # always show your current location in the filesystem: |
|
146 | 146 | |
|
147 | 147 | #o.prompt_in1 = r'\C_LightBlue[\C_LightCyan\Y2\C_LightBlue]\C_Normal\n\C_Green|\#>' |
|
148 | 148 | #o.prompt_in2 = r'.\D: ' |
|
149 | 149 | #o.prompt_out = r'[\#] ' |
|
150 | 150 | |
|
151 | 151 | # Try one of these color settings if you can't read the text easily |
|
152 | 152 | # autoexec is a list of IPython commands to execute on startup |
|
153 | 153 | #o.autoexec.append('%colors LightBG') |
|
154 | 154 | #o.autoexec.append('%colors NoColor') |
|
155 | 155 | o.autoexec.append('%colors Linux') |
|
156 | 156 | |
|
157 | 157 | |
|
158 | 158 | # some config helper functions you can use |
|
159 | 159 | def import_all(modules): |
|
160 | 160 | """ Usage: import_all("os sys") """ |
|
161 | 161 | for m in modules.split(): |
|
162 | 162 | ip.ex("from %s import *" % m) |
|
163 | 163 | |
|
164 | 164 | def execf(fname): |
|
165 | 165 | """ Execute a file in user namespace """ |
|
166 | 166 | ip.ex('execfile("%s")' % os.path.expanduser(fname)) |
|
167 | 167 | |
|
168 | 168 | main() |
|
169 | 169 | |
|
170 | 170 | .. _Prompts: |
|
171 | 171 | |
|
172 | 172 | Fine-tuning your prompt |
|
173 | 173 | ======================= |
|
174 | 174 | |
|
175 | 175 | IPython's prompts can be customized using a syntax similar to that of |
|
176 | 176 | the bash shell. Many of bash's escapes are supported, as well as a few |
|
177 | 177 | additional ones. We list them below:: |
|
178 | 178 | |
|
179 | 179 | \# |
|
180 | 180 | the prompt/history count number. This escape is automatically |
|
181 | 181 | wrapped in the coloring codes for the currently active color scheme. |
|
182 | 182 | \N |
|
183 | 183 | the 'naked' prompt/history count number: this is just the number |
|
184 | 184 | itself, without any coloring applied to it. This lets you produce |
|
185 | 185 | numbered prompts with your own colors. |
|
186 | 186 | \D |
|
187 | 187 | the prompt/history count, with the actual digits replaced by dots. |
|
188 | 188 | Used mainly in continuation prompts (prompt_in2) |
|
189 | 189 | \w |
|
190 | 190 | the current working directory |
|
191 | 191 | \W |
|
192 | 192 | the basename of current working directory |
|
193 | 193 | \Xn |
|
194 | 194 | where $n=0\ldots5.$ The current working directory, with $HOME |
|
195 | 195 | replaced by ~, and filtered out to contain only $n$ path elements |
|
196 | 196 | \Yn |
|
197 | 197 | Similar to \Xn, but with the $n+1$ element included if it is ~ (this |
|
198 | 198 | is similar to the behavior of the %cn escapes in tcsh) |
|
199 | 199 | \u |
|
200 | 200 | the username of the current user |
|
201 | 201 | \$ |
|
202 | 202 | if the effective UID is 0, a #, otherwise a $ |
|
203 | 203 | \h |
|
204 | 204 | the hostname up to the first '.' |
|
205 | 205 | \H |
|
206 | 206 | the hostname |
|
207 | 207 | \n |
|
208 | 208 | a newline |
|
209 | 209 | \r |
|
210 | 210 | a carriage return |
|
211 | 211 | \v |
|
212 | 212 | IPython version string |
|
213 | 213 | |
|
214 | 214 | In addition to these, ANSI color escapes can be insterted into the |
|
215 | 215 | prompts, as \C_ColorName. The list of valid color names is: Black, Blue, |
|
216 | 216 | Brown, Cyan, DarkGray, Green, LightBlue, LightCyan, LightGray, |
|
217 | 217 | LightGreen, LightPurple, LightRed, NoColor, Normal, Purple, Red, White, |
|
218 | 218 | Yellow. |
|
219 | 219 | |
|
220 | 220 | Finally, IPython supports the evaluation of arbitrary expressions in |
|
221 | 221 | your prompt string. The prompt strings are evaluated through the syntax |
|
222 | 222 | of PEP 215, but basically you can use $x.y to expand the value of x.y, |
|
223 | 223 | and for more complicated expressions you can use braces: ${foo()+x} will |
|
224 | 224 | call function foo and add to it the value of x, before putting the |
|
225 | 225 | result into your prompt. For example, using |
|
226 | 226 | prompt_in1 '${commands.getoutput("uptime")}\nIn [\#]: ' |
|
227 | 227 | will print the result of the uptime command on each prompt (assuming the |
|
228 | 228 | commands module has been imported in your ipythonrc file). |
|
229 | 229 | |
|
230 | 230 | |
|
231 | 231 | Prompt examples |
|
232 | 232 | |
|
233 | 233 | The following options in an ipythonrc file will give you IPython's |
|
234 | 234 | default prompts:: |
|
235 | 235 | |
|
236 | 236 | prompt_in1 'In [\#]:' |
|
237 | 237 | prompt_in2 ' .\D.:' |
|
238 | 238 | prompt_out 'Out[\#]:' |
|
239 | 239 | |
|
240 | 240 | which look like this:: |
|
241 | 241 | |
|
242 | 242 | In [1]: 1+2 |
|
243 | 243 | Out[1]: 3 |
|
244 | 244 | |
|
245 | 245 | In [2]: for i in (1,2,3): |
|
246 | 246 | ...: print i, |
|
247 | 247 | ...: |
|
248 | 248 | 1 2 3 |
|
249 | 249 | |
|
250 | 250 | These will give you a very colorful prompt with path information:: |
|
251 | 251 | |
|
252 | 252 | #prompt_in1 '\C_Red\u\C_Blue[\C_Cyan\Y1\C_Blue]\C_LightGreen\#>' |
|
253 | 253 | prompt_in2 ' ..\D>' |
|
254 | 254 | prompt_out '<\#>' |
|
255 | 255 | |
|
256 | 256 | which look like this:: |
|
257 | 257 | |
|
258 | 258 | fperez[~/ipython]1> 1+2 |
|
259 | 259 | <1> 3 |
|
260 | 260 | fperez[~/ipython]2> for i in (1,2,3): |
|
261 | 261 | ...> print i, |
|
262 | 262 | ...> |
|
263 | 263 | 1 2 3 |
|
264 | 264 | |
|
265 | 265 | |
|
266 | 266 | .. _Profiles: |
|
267 | 267 | |
|
268 | 268 | IPython profiles |
|
269 | 269 | ================ |
|
270 | 270 | |
|
271 | 271 | As we already mentioned, IPython supports the -profile command-line option (see |
|
272 | 272 | :ref:`here <command_line_options>`). A profile is nothing more than a |
|
273 | 273 | particular configuration file like your basic ipythonrc one, but with |
|
274 | 274 | particular customizations for a specific purpose. When you start IPython with |
|
275 | 275 | 'ipython -profile <name>', it assumes that in your IPYTHONDIR there is a file |
|
276 | 276 | called ipythonrc-<name> or ipy_profile_<name>.py, and loads it instead of the |
|
277 | 277 | normal ipythonrc. |
|
278 | 278 | |
|
279 | 279 | This system allows you to maintain multiple configurations which load |
|
280 | 280 | modules, set options, define functions, etc. suitable for different |
|
281 | 281 | tasks and activate them in a very simple manner. In order to avoid |
|
282 | 282 | having to repeat all of your basic options (common things that don't |
|
283 | 283 | change such as your color preferences, for example), any profile can |
|
284 | 284 | include another configuration file. The most common way to use profiles |
|
285 | 285 | is then to have each one include your basic ipythonrc file as a starting |
|
286 | point, and then add further customizations. No newline at end of file | |
|
286 | point, and then add further customizations. | |
|
287 |
@@ -1,10 +1,11 b'' | |||
|
1 | 1 | =============================== |
|
2 | 2 | Configuration and customization |
|
3 | 3 | =============================== |
|
4 | 4 | |
|
5 | 5 | .. toctree:: |
|
6 | 6 | :maxdepth: 2 |
|
7 | 7 | |
|
8 | 8 | initial_config.txt |
|
9 | 9 | customization.txt |
|
10 | 10 | new_config.txt |
|
11 |
@@ -1,249 +1,250 b'' | |||
|
1 | 1 | .. _initial config: |
|
2 | 2 | |
|
3 | 3 | ========================================= |
|
4 | 4 | Initial configuration of your environment |
|
5 | 5 | ========================================= |
|
6 | 6 | |
|
7 | 7 | This section will help you set various things in your environment for |
|
8 | 8 | your IPython sessions to be as efficient as possible. All of IPython's |
|
9 | 9 | configuration information, along with several example files, is stored |
|
10 | 10 | in a directory named by default $HOME/.ipython. You can change this by |
|
11 | 11 | defining the environment variable IPYTHONDIR, or at runtime with the |
|
12 | 12 | command line option -ipythondir. |
|
13 | 13 | |
|
14 | 14 | If all goes well, the first time you run IPython it should automatically create |
|
15 | 15 | a user copy of the config directory for you, based on its builtin defaults. You |
|
16 | 16 | can look at the files it creates to learn more about configuring the |
|
17 | 17 | system. The main file you will modify to configure IPython's behavior is called |
|
18 | 18 | ipythonrc (with a .ini extension under Windows), included for reference |
|
19 | 19 | :ref:`here <ipythonrc>`. This file is very commented and has many variables you |
|
20 | 20 | can change to suit your taste, you can find more details :ref:`here |
|
21 | 21 | <customization>`. Here we discuss the basic things you will want to make sure |
|
22 | 22 | things are working properly from the beginning. |
|
23 | 23 | |
|
24 | 24 | |
|
25 | 25 | .. _accessing_help: |
|
26 | 26 | |
|
27 | 27 | Access to the Python help system |
|
28 | 28 | ================================ |
|
29 | 29 | |
|
30 | 30 | This is true for Python in general (not just for IPython): you should have an |
|
31 | 31 | environment variable called PYTHONDOCS pointing to the directory where your |
|
32 | 32 | HTML Python documentation lives. In my system it's |
|
33 | 33 | :file:`/usr/share/doc/python-doc/html`, check your local details or ask your |
|
34 | 34 | systems administrator. |
|
35 | 35 | |
|
36 | 36 | This is the directory which holds the HTML version of the Python |
|
37 | 37 | manuals. Unfortunately it seems that different Linux distributions |
|
38 | 38 | package these files differently, so you may have to look around a bit. |
|
39 | 39 | Below I show the contents of this directory on my system for reference:: |
|
40 | 40 | |
|
41 | 41 | [html]> ls |
|
42 | 42 | about.html dist/ icons/ lib/ python2.5.devhelp.gz whatsnew/ |
|
43 | 43 | acks.html doc/ index.html mac/ ref/ |
|
44 | 44 | api/ ext/ inst/ modindex.html tut/ |
|
45 | 45 | |
|
46 | 46 | You should really make sure this variable is correctly set so that |
|
47 | 47 | Python's pydoc-based help system works. It is a powerful and convenient |
|
48 | 48 | system with full access to the Python manuals and all modules accessible |
|
49 | 49 | to you. |
|
50 | 50 | |
|
51 | 51 | Under Windows it seems that pydoc finds the documentation automatically, |
|
52 | 52 | so no extra setup appears necessary. |
|
53 | 53 | |
|
54 | 54 | |
|
55 | 55 | Editor |
|
56 | 56 | ====== |
|
57 | 57 | |
|
58 | 58 | The %edit command (and its alias %ed) will invoke the editor set in your |
|
59 | 59 | environment as EDITOR. If this variable is not set, it will default to |
|
60 | 60 | vi under Linux/Unix and to notepad under Windows. You may want to set |
|
61 | 61 | this variable properly and to a lightweight editor which doesn't take |
|
62 | 62 | too long to start (that is, something other than a new instance of |
|
63 | 63 | Emacs). This way you can edit multi-line code quickly and with the power |
|
64 | 64 | of a real editor right inside IPython. |
|
65 | 65 | |
|
66 | 66 | If you are a dedicated Emacs user, you should set up the Emacs server so |
|
67 | 67 | that new requests are handled by the original process. This means that |
|
68 | 68 | almost no time is spent in handling the request (assuming an Emacs |
|
69 | 69 | process is already running). For this to work, you need to set your |
|
70 | 70 | EDITOR environment variable to 'emacsclient'. The code below, supplied |
|
71 | 71 | by Francois Pinard, can then be used in your .emacs file to enable the |
|
72 | 72 | server:: |
|
73 | 73 | |
|
74 | 74 | (defvar server-buffer-clients) |
|
75 | 75 | (when (and (fboundp 'server-start) (string-equal (getenv "TERM") 'xterm)) |
|
76 | 76 | (server-start) |
|
77 | 77 | (defun fp-kill-server-with-buffer-routine () |
|
78 | 78 | (and server-buffer-clients (server-done))) |
|
79 | 79 | (add-hook 'kill-buffer-hook 'fp-kill-server-with-buffer-routine)) |
|
80 | 80 | |
|
81 | 81 | You can also set the value of this editor via the commmand-line option |
|
82 | 82 | '-editor' or in your ipythonrc file. This is useful if you wish to use |
|
83 | 83 | specifically for IPython an editor different from your typical default |
|
84 | 84 | (and for Windows users who tend to use fewer environment variables). |
|
85 | 85 | |
|
86 | 86 | |
|
87 | 87 | Color |
|
88 | 88 | ===== |
|
89 | 89 | |
|
90 | 90 | The default IPython configuration has most bells and whistles turned on |
|
91 | 91 | (they're pretty safe). But there's one that may cause problems on some |
|
92 | 92 | systems: the use of color on screen for displaying information. This is |
|
93 | 93 | very useful, since IPython can show prompts and exception tracebacks |
|
94 | 94 | with various colors, display syntax-highlighted source code, and in |
|
95 | 95 | general make it easier to visually parse information. |
|
96 | 96 | |
|
97 | 97 | The following terminals seem to handle the color sequences fine: |
|
98 | 98 | |
|
99 | 99 | * Linux main text console, KDE Konsole, Gnome Terminal, E-term, |
|
100 | 100 | rxvt, xterm. |
|
101 | 101 | * CDE terminal (tested under Solaris). This one boldfaces light colors. |
|
102 | 102 | * (X)Emacs buffers. See the emacs_ section for more details on |
|
103 | 103 | using IPython with (X)Emacs. |
|
104 | 104 | * A Windows (XP/2k) command prompt with pyreadline_. |
|
105 | 105 | * A Windows (XP/2k) CygWin shell. Although some users have reported |
|
106 | 106 | problems; it is not clear whether there is an issue for everyone |
|
107 | 107 | or only under specific configurations. If you have full color |
|
108 | 108 | support under cygwin, please post to the IPython mailing list so |
|
109 | 109 | this issue can be resolved for all users. |
|
110 | 110 | |
|
111 | 111 | .. _pyreadline: https://code.launchpad.net/pyreadline |
|
112 | 112 | |
|
113 | 113 | These have shown problems: |
|
114 | 114 | |
|
115 | 115 | * Windows command prompt in WinXP/2k logged into a Linux machine via |
|
116 | 116 | telnet or ssh. |
|
117 | 117 | * Windows native command prompt in WinXP/2k, without Gary Bishop's |
|
118 | 118 | extensions. Once Gary's readline library is installed, the normal |
|
119 | 119 | WinXP/2k command prompt works perfectly. |
|
120 | 120 | |
|
121 | 121 | Currently the following color schemes are available: |
|
122 | 122 | |
|
123 | 123 | * NoColor: uses no color escapes at all (all escapes are empty '' '' |
|
124 | 124 | strings). This 'scheme' is thus fully safe to use in any terminal. |
|
125 | 125 | * Linux: works well in Linux console type environments: dark |
|
126 | 126 | background with light fonts. It uses bright colors for |
|
127 | 127 | information, so it is difficult to read if you have a light |
|
128 | 128 | colored background. |
|
129 | 129 | * LightBG: the basic colors are similar to those in the Linux scheme |
|
130 | 130 | but darker. It is easy to read in terminals with light backgrounds. |
|
131 | 131 | |
|
132 | 132 | IPython uses colors for two main groups of things: prompts and |
|
133 | 133 | tracebacks which are directly printed to the terminal, and the object |
|
134 | 134 | introspection system which passes large sets of data through a pager. |
|
135 | 135 | |
|
136 | 136 | |
|
137 | 137 | Input/Output prompts and exception tracebacks |
|
138 | 138 | ============================================= |
|
139 | 139 | |
|
140 | 140 | You can test whether the colored prompts and tracebacks work on your |
|
141 | 141 | system interactively by typing '%colors Linux' at the prompt (use |
|
142 | 142 | '%colors LightBG' if your terminal has a light background). If the input |
|
143 | 143 | prompt shows garbage like:: |
|
144 | 144 | |
|
145 | 145 | [0;32mIn [[1;32m1[0;32m]: [0;00m |
|
146 | 146 | |
|
147 | 147 | instead of (in color) something like:: |
|
148 | 148 | |
|
149 | 149 | In [1]: |
|
150 | 150 | |
|
151 | 151 | this means that your terminal doesn't properly handle color escape |
|
152 | 152 | sequences. You can go to a 'no color' mode by typing '%colors NoColor'. |
|
153 | 153 | |
|
154 | 154 | You can try using a different terminal emulator program (Emacs users, |
|
155 | 155 | see below). To permanently set your color preferences, edit the file |
|
156 | 156 | $HOME/.ipython/ipythonrc and set the colors option to the desired value. |
|
157 | 157 | |
|
158 | 158 | |
|
159 | 159 | Object details (types, docstrings, source code, etc.) |
|
160 | 160 | ===================================================== |
|
161 | 161 | |
|
162 | 162 | IPython has a set of special functions for studying the objects you are working |
|
163 | 163 | with, discussed in detail :ref:`here <dynamic_object_info>`. But this system |
|
164 | 164 | relies on passing information which is longer than your screen through a data |
|
165 | 165 | pager, such as the common Unix less and more programs. In order to be able to |
|
166 | 166 | see this information in color, your pager needs to be properly configured. I |
|
167 | 167 | strongly recommend using less instead of more, as it seems that more simply can |
|
168 | 168 | not understand colored text correctly. |
|
169 | 169 | |
|
170 | 170 | In order to configure less as your default pager, do the following: |
|
171 | 171 | |
|
172 | 172 | 1. Set the environment PAGER variable to less. |
|
173 | 173 | 2. Set the environment LESS variable to -r (plus any other options |
|
174 | 174 | you always want to pass to less by default). This tells less to |
|
175 | 175 | properly interpret control sequences, which is how color |
|
176 | 176 | information is given to your terminal. |
|
177 | 177 | |
|
178 | 178 | For the bash shell, add to your ~/.bashrc file the lines:: |
|
179 | 179 | |
|
180 | 180 | export PAGER=less |
|
181 | 181 | export LESS=-r |
|
182 | 182 | |
|
183 | 183 | For the csh or tcsh shells, add to your ~/.cshrc file the lines:: |
|
184 | 184 | |
|
185 | 185 | setenv PAGER less |
|
186 | 186 | setenv LESS -r |
|
187 | 187 | |
|
188 | 188 | There is similar syntax for other Unix shells, look at your system |
|
189 | 189 | documentation for details. |
|
190 | 190 | |
|
191 | 191 | If you are on a system which lacks proper data pagers (such as Windows), |
|
192 | 192 | IPython will use a very limited builtin pager. |
|
193 | 193 | |
|
194 | 194 | .. _emacs: |
|
195 | 195 | |
|
196 | 196 | (X)Emacs configuration |
|
197 | 197 | ====================== |
|
198 | 198 | |
|
199 | 199 | Thanks to the work of Alexander Schmolck and Prabhu Ramachandran, |
|
200 | 200 | currently (X)Emacs and IPython get along very well. |
|
201 | 201 | |
|
202 | 202 | Important note: You will need to use a recent enough version of |
|
203 | 203 | python-mode.el, along with the file ipython.el. You can check that the |
|
204 | 204 | version you have of python-mode.el is new enough by either looking at |
|
205 | 205 | the revision number in the file itself, or asking for it in (X)Emacs via |
|
206 | 206 | M-x py-version. Versions 4.68 and newer contain the necessary fixes for |
|
207 | 207 | proper IPython support. |
|
208 | 208 | |
|
209 | 209 | The file ipython.el is included with the IPython distribution, in the |
|
210 | 210 | documentation directory (where this manual resides in PDF and HTML |
|
211 | 211 | formats). |
|
212 | 212 | |
|
213 | 213 | Once you put these files in your Emacs path, all you need in your .emacs |
|
214 | 214 | file is:: |
|
215 | 215 | |
|
216 | 216 | (require 'ipython) |
|
217 | 217 | |
|
218 | 218 | This should give you full support for executing code snippets via |
|
219 | 219 | IPython, opening IPython as your Python shell via ``C-c !``, etc. |
|
220 | 220 | |
|
221 | 221 | You can customize the arguments passed to the IPython instance at startup by |
|
222 | 222 | setting the ``py-python-command-args`` variable. For example, to start always |
|
223 | 223 | in ``pylab`` mode with hardcoded light-background colors, you can use:: |
|
224 | 224 | |
|
225 | 225 | (setq py-python-command-args '("-pylab" "-colors" "LightBG")) |
|
226 | 226 | |
|
227 | 227 | If you happen to get garbage instead of colored prompts as described in |
|
228 | 228 | the previous section, you may need to set also in your .emacs file:: |
|
229 | 229 | |
|
230 | 230 | (setq ansi-color-for-comint-mode t) |
|
231 | 231 | |
|
232 | 232 | Notes: |
|
233 | 233 | |
|
234 | 234 | * There is one caveat you should be aware of: you must start the |
|
235 | 235 | IPython shell before attempting to execute any code regions via |
|
236 | 236 | ``C-c |``. Simply type C-c ! to start IPython before passing any code |
|
237 | 237 | regions to the interpreter, and you shouldn't experience any |
|
238 | 238 | problems. |
|
239 | 239 | This is due to a bug in Python itself, which has been fixed for |
|
240 | 240 | Python 2.3, but exists as of Python 2.2.2 (reported as SF bug [ |
|
241 | 241 | 737947 ]). |
|
242 | 242 | * The (X)Emacs support is maintained by Alexander Schmolck, so all |
|
243 | 243 | comments/requests should be directed to him through the IPython |
|
244 | 244 | mailing lists. |
|
245 | 245 | * This code is still somewhat experimental so it's a bit rough |
|
246 | 246 | around the edges (although in practice, it works quite well). |
|
247 | 247 | * Be aware that if you customize py-python-command previously, this |
|
248 | 248 | value will override what ipython.el does (because loading the |
|
249 | 249 | customization variables comes later). |
|
250 |
@@ -1,27 +1,28 b'' | |||
|
1 | 1 | ======================== |
|
2 | 2 | New configuration system |
|
3 | 3 | ======================== |
|
4 | 4 | |
|
5 | 5 | IPython has a configuration system. When running IPython for the first time, |
|
6 | 6 | reasonable defaults are used for the configuration. The configuration of IPython |
|
7 | 7 | can be changed in two ways: |
|
8 | 8 | |
|
9 | 9 | * Configuration files |
|
10 | 10 | * Commands line options (which override the configuration files) |
|
11 | 11 | |
|
12 | 12 | IPython has a separate configuration file for each subpackage. Thus, the main |
|
13 | 13 | configuration files are (in your ``~/.ipython`` directory): |
|
14 | 14 | |
|
15 | 15 | * ``ipython1.core.ini`` |
|
16 | 16 | * ``ipython1.kernel.ini`` |
|
17 | 17 | * ``ipython1.notebook.ini`` |
|
18 | 18 | |
|
19 | 19 | To create these files for the first time, do the following:: |
|
20 | 20 | |
|
21 | 21 | from IPython.kernel.config import config_manager as kernel_config |
|
22 | 22 | kernel_config.write_default_config_file() |
|
23 | 23 | |
|
24 | 24 | But, you should only need to do this if you need to modify the defaults. If needed |
|
25 | 25 | repeat this process with the ``notebook`` and ``core`` configuration as well. If you |
|
26 | 26 | are running into problems with IPython, you might try deleting these configuration |
|
27 | files. No newline at end of file | |
|
27 | files. | |
|
28 |
@@ -1,141 +1,142 b'' | |||
|
1 | 1 | ============== |
|
2 | 2 | Coding guide |
|
3 | 3 | ============== |
|
4 | 4 | |
|
5 | 5 | |
|
6 | 6 | Coding conventions |
|
7 | 7 | ================== |
|
8 | 8 | |
|
9 | 9 | In general, we'll try to follow the standard Python style conventions as |
|
10 | 10 | described in Python's `PEP 8`_, the official Python Style Guide. |
|
11 | 11 | |
|
12 | 12 | .. _PEP 8: http://www.python.org/peps/pep-0008.html |
|
13 | 13 | |
|
14 | 14 | Other comments: |
|
15 | 15 | |
|
16 | 16 | - In a large file, top level classes and functions should be separated by 2-3 |
|
17 | 17 | lines to make it easier to separate them visually. |
|
18 | 18 | |
|
19 | 19 | - Use 4 spaces for indentation, *never* use hard tabs. |
|
20 | 20 | |
|
21 | 21 | - Keep the ordering of methods the same in classes that have the same methods. |
|
22 | 22 | This is particularly true for classes that implement similar interfaces and |
|
23 | 23 | for interfaces that are similar. |
|
24 | 24 | |
|
25 | 25 | Naming conventions |
|
26 | 26 | ------------------ |
|
27 | 27 | |
|
28 | 28 | In terms of naming conventions, we'll follow the guidelines of PEP 8. Some of |
|
29 | 29 | the existing code doesn't honor this perfectly, but for all new IPython code |
|
30 | 30 | (and much existing code is being refactored), we'll use: |
|
31 | 31 | |
|
32 | 32 | - All ``lowercase`` module names. |
|
33 | 33 | |
|
34 | 34 | - ``CamelCase`` for class names. |
|
35 | 35 | |
|
36 | 36 | - ``lowercase_with_underscores`` for methods, functions, variables and |
|
37 | 37 | attributes. |
|
38 | 38 | |
|
39 | 39 | This may be confusing as some of the existing codebase uses a different |
|
40 | 40 | convention (``lowerCamelCase`` for methods and attributes). Slowly, we will |
|
41 | 41 | move IPython over to the new convention, providing shadow names for backward |
|
42 | 42 | compatibility in public interfaces. |
|
43 | 43 | |
|
44 | 44 | There are, however, some important exceptions to these rules. In some cases, |
|
45 | 45 | IPython code will interface with packages (Twisted, Wx, Qt) that use other |
|
46 | 46 | conventions. At some level this makes it impossible to adhere to our own |
|
47 | 47 | standards at all times. In particular, when subclassing classes that use other |
|
48 | 48 | naming conventions, you must follow their naming conventions. To deal with |
|
49 | 49 | cases like this, we propose the following policy: |
|
50 | 50 | |
|
51 | 51 | - If you are subclassing a class that uses different conventions, use its |
|
52 | 52 | naming conventions throughout your subclass. Thus, if you are creating a |
|
53 | 53 | Twisted Protocol class, used Twisted's |
|
54 | 54 | ``namingSchemeForMethodsAndAttributes.`` |
|
55 | 55 | |
|
56 | 56 | - All IPython's official interfaces should use our conventions. In some cases |
|
57 | 57 | this will mean that you need to provide shadow names (first implement |
|
58 | 58 | ``fooBar`` and then ``foo_bar = fooBar``). We want to avoid this at all |
|
59 | 59 | costs, but it will probably be necessary at times. But, please use this |
|
60 | 60 | sparingly! |
|
61 | 61 | |
|
62 | 62 | Implementation-specific *private* methods will use |
|
63 | 63 | ``_single_underscore_prefix``. Names with a leading double underscore will |
|
64 | 64 | *only* be used in special cases, as they makes subclassing difficult (such |
|
65 | 65 | names are not easily seen by child classes). |
|
66 | 66 | |
|
67 | 67 | Occasionally some run-in lowercase names are used, but mostly for very short |
|
68 | 68 | names or where we are implementing methods very similar to existing ones in a |
|
69 | 69 | base class (like ``runlines()`` where ``runsource()`` and ``runcode()`` had |
|
70 | 70 | established precedent). |
|
71 | 71 | |
|
72 | 72 | The old IPython codebase has a big mix of classes and modules prefixed with an |
|
73 | 73 | explicit ``IP``. In Python this is mostly unnecessary, redundant and frowned |
|
74 | 74 | upon, as namespaces offer cleaner prefixing. The only case where this approach |
|
75 | 75 | is justified is for classes which are expected to be imported into external |
|
76 | 76 | namespaces and a very generic name (like Shell) is too likely to clash with |
|
77 | 77 | something else. We'll need to revisit this issue as we clean up and refactor |
|
78 | 78 | the code, but in general we should remove as many unnecessary ``IP``/``ip`` |
|
79 | 79 | prefixes as possible. However, if a prefix seems absolutely necessary the more |
|
80 | 80 | specific ``IPY`` or ``ipy`` are preferred. |
|
81 | 81 | |
|
82 | 82 | |
|
83 | 83 | .. _devel-testing: |
|
84 | 84 | |
|
85 | 85 | Testing system |
|
86 | 86 | ============== |
|
87 | 87 | |
|
88 | 88 | It is extremely important that all code contributed to IPython has tests. Tests |
|
89 | 89 | should be written as unittests, doctests or as entities that the `Nose`_ |
|
90 | 90 | testing package will find. Regardless of how the tests are written, we will use |
|
91 | 91 | `Nose`_ for discovering and running the tests. `Nose`_ will be required to run |
|
92 | 92 | the IPython test suite, but will not be required to simply use IPython. |
|
93 | 93 | |
|
94 | 94 | .. _Nose: http://code.google.com/p/python-nose/ |
|
95 | 95 | |
|
96 | 96 | Tests of `Twisted`__ using code should be written by subclassing the |
|
97 | 97 | ``TestCase`` class that comes with ``twisted.trial.unittest``. When this is |
|
98 | 98 | done, `Nose`_ will be able to run the tests and the twisted reactor will be |
|
99 | 99 | handled correctly. |
|
100 | 100 | |
|
101 | 101 | .. __: http://www.twistedmatrix.com |
|
102 | 102 | |
|
103 | 103 | Each subpackage in IPython should have its own ``tests`` directory that |
|
104 | 104 | contains all of the tests for that subpackage. This allows each subpackage to |
|
105 | 105 | be self-contained. If a subpackage has any dependencies beyond the Python |
|
106 | 106 | standard library, the tests for that subpackage should be skipped if the |
|
107 | 107 | dependencies are not found. This is very important so users don't get tests |
|
108 | 108 | failing simply because they don't have dependencies. |
|
109 | 109 | |
|
110 | 110 | We also need to look into use Noses ability to tag tests to allow a more |
|
111 | 111 | modular approach of running tests. |
|
112 | 112 | |
|
113 | 113 | .. _devel-config: |
|
114 | 114 | |
|
115 | 115 | Configuration system |
|
116 | 116 | ==================== |
|
117 | 117 | |
|
118 | 118 | IPython uses `.ini`_ files for configuration purposes. This represents a huge |
|
119 | 119 | improvement over the configuration system used in IPython. IPython works with |
|
120 | 120 | these files using the `ConfigObj`_ package, which IPython includes as |
|
121 | 121 | ``ipython1/external/configobj.py``. |
|
122 | 122 | |
|
123 | 123 | Currently, we are using raw `ConfigObj`_ objects themselves. Each subpackage of |
|
124 | 124 | IPython should contain a ``config`` subdirectory that contains all of the |
|
125 | 125 | configuration information for the subpackage. To see how configuration |
|
126 | 126 | information is defined (along with defaults) see at the examples in |
|
127 | 127 | ``ipython1/kernel/config`` and ``ipython1/core/config``. Likewise, to see how |
|
128 | 128 | the configuration information is used, see examples in |
|
129 | 129 | ``ipython1/kernel/scripts/ipengine.py``. |
|
130 | 130 | |
|
131 | 131 | Eventually, we will add a new layer on top of the raw `ConfigObj`_ objects. We |
|
132 | 132 | are calling this new layer, ``tconfig``, as it will use a `Traits`_-like |
|
133 | 133 | validation model. We won't actually use `Traits`_, but will implement |
|
134 | 134 | something similar in pure Python. But, even in this new system, we will still |
|
135 | 135 | use `ConfigObj`_ and `.ini`_ files underneath the hood. Talk to Fernando if you |
|
136 | 136 | are interested in working on this part of IPython. The current prototype of |
|
137 | 137 | ``tconfig`` is located in the IPython sandbox. |
|
138 | 138 | |
|
139 | 139 | .. _.ini: http://docs.python.org/lib/module-ConfigParser.html |
|
140 | 140 | .. _ConfigObj: http://www.voidspace.org.uk/python/configobj.html |
|
141 | 141 | .. _Traits: http://code.enthought.com/traits/ |
|
142 |
@@ -1,33 +1,34 b'' | |||
|
1 | 1 | ========================================= |
|
2 | 2 | Notes on the IPython configuration system |
|
3 | 3 | ========================================= |
|
4 | 4 | |
|
5 | 5 | This document has some random notes on the configuration system. |
|
6 | 6 | |
|
7 | 7 | To start, an IPython process needs: |
|
8 | 8 | |
|
9 | 9 | * Configuration files |
|
10 | 10 | * Command line options |
|
11 | 11 | * Additional files (FURL files, extra scripts, etc.) |
|
12 | 12 | |
|
13 | 13 | It feeds these things into the core logic of the process, and as output, |
|
14 | 14 | produces: |
|
15 | 15 | |
|
16 | 16 | * Log files |
|
17 | 17 | * Security files |
|
18 | 18 | |
|
19 | 19 | There are a number of things that complicate this: |
|
20 | 20 | |
|
21 | 21 | * A process may need to be started on a different host that doesn't have |
|
22 | 22 | any of the config files or additional files. Those files need to be |
|
23 | 23 | moved over and put in a staging area. The process then needs to be told |
|
24 | 24 | about them. |
|
25 | 25 | * The location of the output files should somehow be set by config files or |
|
26 | 26 | command line options. |
|
27 | 27 | * Our config files are very hierarchical, but command line options are flat, |
|
28 | 28 | making it difficult to relate command line options to config files. |
|
29 | 29 | * Some processes (like ipcluster and the daemons) have to manage the input and |
|
30 | 30 | output files for multiple different subprocesses, each possibly on a |
|
31 | 31 | different host. Ahhhh! |
|
32 | 32 | * Our configurations are not singletons. A given user will likely have |
|
33 | 33 | many different configurations for different clusters. |
|
34 |
@@ -1,103 +1,104 b'' | |||
|
1 | 1 | .. _documenting-ipython: |
|
2 | 2 | |
|
3 | 3 | ===================== |
|
4 | 4 | Documenting IPython |
|
5 | 5 | ===================== |
|
6 | 6 | |
|
7 | 7 | Standalone documentation |
|
8 | 8 | ======================== |
|
9 | 9 | |
|
10 | 10 | All standalone documentation should be written in plain text (``.txt``) files |
|
11 | 11 | using `reStructuredText`_ for markup and formatting. All such documentation |
|
12 | 12 | should be placed in the top level directory ``docs`` of the IPython source |
|
13 | 13 | tree. Or, when appropriate, a suitably named subdirectory should be used. The |
|
14 | 14 | documentation in this location will serve as the main source for IPython |
|
15 | 15 | documentation and all existing documentation should be converted to this |
|
16 | 16 | format. |
|
17 | 17 | |
|
18 | 18 | The actual HTML and PDF docs are built using the Sphinx_ documentation |
|
19 | 19 | generation tool. Sphinx has been adopted as the default documentation tool for |
|
20 | 20 | Python itself as of version 2.6, as well as by a number of projects that |
|
21 | 21 | IPython is related with, such as numpy, scipy, matplotlib, sage and nipy. |
|
22 | 22 | |
|
23 | 23 | .. _reStructuredText: http://docutils.sourceforge.net/rst.html |
|
24 | 24 | .. _Sphinx: http://sphinx.pocoo.org/ |
|
25 | 25 | |
|
26 | 26 | |
|
27 | 27 | The rest of this document is mostly taken from the `matploblib |
|
28 | 28 | documentation`__; we are using a number of Sphinx tools and extensions written |
|
29 | 29 | by the matplotlib team and will mostly follow their conventions, which are |
|
30 | 30 | nicely spelled out in their guide. What follows is thus a lightly adapted |
|
31 | 31 | version of the matplotlib documentation guide, taken with permission from the |
|
32 | 32 | MPL team. |
|
33 | 33 | |
|
34 | 34 | .. __: http://matplotlib.sourceforge.net/devel/documenting_mpl.html |
|
35 | 35 | |
|
36 | 36 | |
|
37 | 37 | A bit of Python code:: |
|
38 | 38 | |
|
39 | 39 | for i in range(10): |
|
40 | 40 | print i, |
|
41 | 41 | print "A big number:",2**34 |
|
42 | 42 | |
|
43 | 43 | An interactive Python session:: |
|
44 | 44 | |
|
45 | 45 | >>> from IPython import genutils |
|
46 | 46 | >>> genutils.get_ipython_dir() |
|
47 | 47 | '/home/fperez/.ipython' |
|
48 | 48 | |
|
49 | 49 | |
|
50 | 50 | An IPython session: |
|
51 | 51 | |
|
52 | 52 | .. code-block:: ipython |
|
53 | 53 | |
|
54 | 54 | In [7]: import IPython |
|
55 | 55 | |
|
56 | 56 | In [8]: print "This IPython is version:",IPython.__version__ |
|
57 | 57 | This IPython is version: 0.9.1 |
|
58 | 58 | |
|
59 | 59 | In [9]: 2+4 |
|
60 | 60 | Out[9]: 6 |
|
61 | 61 | |
|
62 | 62 | |
|
63 | 63 | A bit of shell code: |
|
64 | 64 | |
|
65 | 65 | .. code-block:: bash |
|
66 | 66 | |
|
67 | 67 | cd /tmp |
|
68 | 68 | echo "My home directory is: $HOME" |
|
69 | 69 | ls |
|
70 | 70 | |
|
71 | 71 | |
|
72 | 72 | Docstring format |
|
73 | 73 | ================ |
|
74 | 74 | |
|
75 | 75 | Good docstrings are very important. Unfortunately, Python itself only provides |
|
76 | 76 | a rather loose standard for docstrings (`PEP 257`_), and there is no universally |
|
77 | 77 | accepted convention for all the different parts of a complete docstring. |
|
78 | 78 | However, the NumPy project has established a very reasonable standard, and has |
|
79 | 79 | developed some tools to support the smooth inclusion of such docstrings in |
|
80 | 80 | Sphinx-generated manuals. Rather than inventing yet another pseudo-standard, |
|
81 | 81 | IPython will be henceforth documented using the NumPy conventions; we carry |
|
82 | 82 | copies of some of the NumPy support tools to remain self-contained, but share |
|
83 | 83 | back upstream with NumPy any improvements or fixes we may make to the tools. |
|
84 | 84 | |
|
85 | 85 | The `NumPy documentation guidelines`_ contain detailed information on this |
|
86 | 86 | standard, and for a quick overview, the NumPy `example docstring`_ is a useful |
|
87 | 87 | read. |
|
88 | 88 | |
|
89 | 89 | As in the past IPython used epydoc, currently many docstrings still use epydoc |
|
90 | 90 | conventions. We will update them as we go, but all new code should be fully |
|
91 | 91 | documented using the NumPy standard. |
|
92 | 92 | |
|
93 | 93 | .. _PEP 257: http://www.python.org/peps/pep-0257.html |
|
94 | 94 | .. _NumPy documentation guidelines: http://projects.scipy.org/numpy/wiki/CodingStyleGuidelines |
|
95 | 95 | |
|
96 | 96 | .. _example docstring: http://projects.scipy.org/numpy/browser/trunk/doc/EXAMPLE_DOCSTRING.txt |
|
97 | 97 | |
|
98 | 98 | Additional PEPs of interest regarding documentation of code. While both of |
|
99 | 99 | these were rejected, the ideas therein form much of the basis of docutils (the |
|
100 | 100 | machinery to process reStructuredText): |
|
101 | 101 | |
|
102 | 102 | - `Docstring Processing System Framework <http://www.python.org/peps/pep-0256.html>`_ |
|
103 | 103 | - `Docutils Design Specification <http://www.python.org/peps/pep-0258.html>`_ |
|
104 |
@@ -1,15 +1,16 b'' | |||
|
1 | 1 | =========================== |
|
2 | 2 | IPython Developer's Guide |
|
3 | 3 | =========================== |
|
4 | 4 | |
|
5 | 5 | .. toctree:: |
|
6 | 6 | :maxdepth: 2 |
|
7 | 7 | |
|
8 | 8 | overview.txt |
|
9 | 9 | coding_guide.txt |
|
10 | 10 | doc_guide.txt |
|
11 | 11 | roadmap.txt |
|
12 | 12 | |
|
13 | 13 | notification_blueprint.txt |
|
14 | 14 | config_blueprint.txt |
|
15 | 15 | reorg.txt |
|
16 |
@@ -1,83 +1,108 b'' | |||
|
1 | 1 | .. _notification: |
|
2 | 2 | |
|
3 | 3 | ========================================== |
|
4 | 4 | IPython.kernel.core.notification blueprint |
|
5 | 5 | ========================================== |
|
6 | 6 | |
|
7 | 7 | Overview |
|
8 | 8 | ======== |
|
9 | 9 | |
|
10 | 10 | The :mod:`IPython.kernel.core.notification` module will provide a simple |
|
11 | 11 | implementation of a notification center and support for the observer pattern |
|
12 | 12 | within the :mod:`IPython.kernel.core`. The main intended use case is to |
|
13 | 13 | provide notification of Interpreter events to an observing frontend during the |
|
14 | 14 | execution of a single block of code. |
|
15 | 15 | |
|
16 | 16 | Functional Requirements |
|
17 | 17 | ======================= |
|
18 | 18 | |
|
19 | 19 | The notification center must: |
|
20 | 20 | |
|
21 | 21 | * Provide synchronous notification of events to all registered observers. |
|
22 | 22 | |
|
23 | 23 | * Provide typed or labeled notification types. |
|
24 | 24 | |
|
25 | 25 | * Allow observers to register callbacks for individual or all notification |
|
26 | 26 | types. |
|
27 | 27 | |
|
28 | 28 | * Allow observers to register callbacks for events from individual or all |
|
29 | 29 | notifying objects. |
|
30 | 30 | |
|
31 | 31 | * Notification to the observer consists of the notification type, notifying |
|
32 | 32 | object and user-supplied extra information [implementation: as keyword |
|
33 | 33 | parameters to the registered callback]. |
|
34 | 34 | |
|
35 | 35 | * Perform as O(1) in the case of no registered observers. |
|
36 | 36 | |
|
37 | 37 | * Permit out-of-process or cross-network extension. |
|
38 | 38 | |
|
39 | 39 | What's not included |
|
40 | 40 | =================== |
|
41 | 41 | |
|
42 | 42 | As written, the :mod:`IPython.kernel.core.notificaiton` module does not: |
|
43 | 43 | |
|
44 | 44 | * Provide out-of-process or network notifications (these should be handled by |
|
45 | 45 | a separate, Twisted aware module in :mod:`IPython.kernel`). |
|
46 | 46 | |
|
47 | 47 | * Provide zope.interface-style interfaces for the notification system (these |
|
48 | 48 | should also be provided by the :mod:`IPython.kernel` module). |
|
49 | 49 | |
|
50 | 50 | Use Cases |
|
51 | 51 | ========= |
|
52 | 52 | |
|
53 | 53 | The following use cases describe the main intended uses of the notificaiton module and illustrate the main success scenario for each use case: |
|
54 | 54 | |
|
55 | 55 | 1. Dwight Schroot is writing a frontend for the IPython project. His frontend is stuck in the stone age and must communicate synchronously with an IPython.kernel.core.Interpreter instance. Because code is executed in blocks by the Interpreter, Dwight's UI freezes every time he executes a long block of code. To keep track of the progress of his long running block, Dwight adds the following code to his frontend's set-up code:: |
|
56 | 56 | |
|
57 | 57 | from IPython.kernel.core.notification import NotificationCenter |
|
58 | 58 | center = NotificationCenter.sharedNotificationCenter |
|
59 | 59 | center.registerObserver(self, type=IPython.kernel.core.Interpreter.STDOUT_NOTIFICATION_TYPE, notifying_object=self.interpreter, callback=self.stdout_notification) |
|
60 | 60 | |
|
61 | 61 | and elsewhere in his front end:: |
|
62 | 62 | |
|
63 | 63 | def stdout_notification(self, type, notifying_object, out_string=None): |
|
64 | 64 | self.writeStdOut(out_string) |
|
65 | 65 | |
|
66 | 66 | If everything works, the Interpreter will (according to its published API) |
|
67 | 67 | fire a notification via the |
|
68 | 68 | :data:`IPython.kernel.core.notification.sharedCenter` of type |
|
69 | 69 | :const:`STD_OUT_NOTIFICATION_TYPE` before writing anything to stdout [it's up |
|
70 | 70 | to the Intereter implementation to figure out when to do this]. The |
|
71 | 71 | notificaiton center will then call the registered callbacks for that event |
|
72 | 72 | type (in this case, Dwight's frontend's stdout_notification method). Again, |
|
73 | 73 | according to its API, the Interpreter provides an additional keyword argument |
|
74 | 74 | when firing the notificaiton of out_string, a copy of the string it will write |
|
75 | 75 | to stdout. |
|
76 | 76 | |
|
77 | 77 | Like magic, Dwight's frontend is able to provide output, even during |
|
78 | 78 | long-running calculations. Now if Jim could just convince Dwight to use |
|
79 | 79 | Twisted... |
|
80 | 80 | |
|
81 | 2. Boss Hog is writing a frontend for the IPython project. Because Boss Hog is stuck in the stone age, his frontend will be written in a new Fortran-like dialect of python and will run only from the command line. Because he doesn't need any fancy notification system and is used to worrying about every cycle on his rat-wheel powered mini, Boss Hog is adamant that the new notification system not produce any performance penalty. As they say in Hazard county, there's no such thing as a free lunch. If he wanted zero overhead, he should have kept using IPython 0.8. Instead, those tricky Duke boys slide in a suped-up bridge-out jumpin' awkwardly confederate-lovin' notification module that imparts only a constant (and small) performance penalty when the Interpreter (or any other object) fires an event for which there are no registered observers. Of course, the same notificaiton-enabled Interpreter can then be used in frontends that require notifications, thus saving the IPython project from a nasty civil war. | |
|
81 | 2. Boss Hog is writing a frontend for the IPython project. Because Boss Hog is | |
|
82 | stuck in the stone age, his frontend will be written in a new Fortran-like | |
|
83 | dialect of python and will run only from the command line. Because he doesn't | |
|
84 | need any fancy notification system and is used to worrying about every cycle | |
|
85 | on his rat-wheel powered mini, Boss Hog is adamant that the new notification | |
|
86 | system not produce any performance penalty. As they say in Hazard county, | |
|
87 | there's no such thing as a free lunch. If he wanted zero overhead, he should | |
|
88 | have kept using IPython 0.8. Instead, those tricky Duke boys slide in a | |
|
89 | suped-up bridge-out jumpin' awkwardly confederate-lovin' notification module | |
|
90 | that imparts only a constant (and small) performance penalty when the | |
|
91 | Interpreter (or any other object) fires an event for which there are no | |
|
92 | registered observers. Of course, the same notificaiton-enabled Interpreter can | |
|
93 | then be used in frontends that require notifications, thus saving the IPython | |
|
94 | project from a nasty civil war. | |
|
82 | 95 | |
|
83 | 3. Barry is wrting a frontend for the IPython project. Because Barry's front end is the *new hotness*, it uses an asynchronous event model to communicate with a Twisted :mod:`~IPython.kernel.engineservice` that communicates with the IPython :class:`~IPython.kernel.core.interpreter.Interpreter`. Using the :mod:`IPython.kernel.notification` module, an asynchronous wrapper on the :mod:`IPython.kernel.core.notification` module, Barry's frontend can register for notifications from the interpreter that are delivered asynchronously. Even if Barry's frontend is running on a separate process or even host from the Interpreter, the notifications are delivered, as if by dark and twisted magic. Just like Dwight's frontend, Barry's frontend can now recieve notifications of e.g. writing to stdout/stderr, opening/closing an external file, an exception in the executing code, etc. No newline at end of file | |
|
96 | 3. Barry is wrting a frontend for the IPython project. Because Barry's front | |
|
97 | end is the *new hotness*, it uses an asynchronous event model to communicate | |
|
98 | with a Twisted :mod:`~IPython.kernel.engineservice` that communicates with the | |
|
99 | IPython :class:`~IPython.kernel.core.interpreter.Interpreter`. Using the | |
|
100 | :mod:`IPython.kernel.notification` module, an asynchronous wrapper on the | |
|
101 | :mod:`IPython.kernel.core.notification` module, Barry's frontend can register | |
|
102 | for notifications from the interpreter that are delivered asynchronously. Even | |
|
103 | if Barry's frontend is running on a separate process or even host from the | |
|
104 | Interpreter, the notifications are delivered, as if by dark and twisted magic. | |
|
105 | Just like Dwight's frontend, Barry's frontend can now recieve notifications of | |
|
106 | e.g. writing to stdout/stderr, opening/closing an external file, an exception | |
|
107 | in the executing code, etc. | |
|
108 |
@@ -1,517 +1,518 b'' | |||
|
1 | 1 | .. _development: |
|
2 | 2 | |
|
3 | 3 | ============================== |
|
4 | 4 | IPython development guidelines |
|
5 | 5 | ============================== |
|
6 | 6 | |
|
7 | 7 | |
|
8 | 8 | Overview |
|
9 | 9 | ======== |
|
10 | 10 | |
|
11 | 11 | This document describes IPython from the perspective of developers. Most |
|
12 | 12 | importantly, it gives information for people who want to contribute to the |
|
13 | 13 | development of IPython. So if you want to help out, read on! |
|
14 | 14 | |
|
15 | 15 | How to contribute to IPython |
|
16 | 16 | ============================ |
|
17 | 17 | |
|
18 | 18 | IPython development is done using Bazaar [Bazaar]_ and Launchpad [Launchpad]_. |
|
19 | 19 | This makes it easy for people to contribute to the development of IPython. |
|
20 | 20 | There are several ways in which you can join in. |
|
21 | 21 | |
|
22 | 22 | If you have a small change that you want to send to the team, you can edit your |
|
23 | 23 | bazaar checkout of IPython (see below) in-place, and ask bazaar for the |
|
24 | 24 | differences:: |
|
25 | 25 | |
|
26 | 26 | $ cd /path/to/your/copy/of/ipython |
|
27 | 27 | $ bzr diff > my_fixes.diff |
|
28 | 28 | |
|
29 | 29 | This produces a patch file with your fixes, which we can apply to the source |
|
30 | 30 | tree. This file should then be attached to a ticket in our `bug tracker |
|
31 | 31 | <https://bugs.launchpad.net/ipython>`_, indicating what it does. |
|
32 | 32 | |
|
33 | 33 | This model of creating small, self-contained patches works very well and there |
|
34 | 34 | are open source projects that do their entire development this way. However, |
|
35 | 35 | in IPython we have found that for tracking larger changes, making use of |
|
36 | 36 | bazaar's full capabilities in conjunction with Launchpad's code hosting |
|
37 | 37 | services makes for a much better experience. |
|
38 | 38 | |
|
39 | 39 | Making your own branch of IPython allows you to refine your changes over time, |
|
40 | 40 | track the development of the main team, and propose your own full version of |
|
41 | 41 | the code for others to use and review, with a minimum amount of fuss. The next |
|
42 | 42 | parts of this document will explain how to do this. |
|
43 | 43 | |
|
44 | 44 | Install Bazaar and create a Launchpad account |
|
45 | 45 | --------------------------------------------- |
|
46 | 46 | |
|
47 | 47 | First make sure you have installed Bazaar (see their `website |
|
48 | 48 | <http://bazaar-vcs.org/>`_). To see that Bazaar is installed and knows about |
|
49 | 49 | you, try the following:: |
|
50 | 50 | |
|
51 | 51 | $ bzr whoami |
|
52 | 52 | Joe Coder <jcoder@gmail.com> |
|
53 | 53 | |
|
54 | 54 | This should display your name and email. Next, you will want to create an |
|
55 | 55 | account on the `Launchpad website <http://www.launchpad.net>`_ and setup your |
|
56 | 56 | ssh keys. For more information of setting up your ssh keys, see `this link |
|
57 | 57 | <https://help.launchpad.net/YourAccount/CreatingAnSSHKeyPair>`_. |
|
58 | 58 | |
|
59 | 59 | Get the main IPython branch from Launchpad |
|
60 | 60 | ------------------------------------------ |
|
61 | 61 | |
|
62 | 62 | Now, you can get a copy of the main IPython development branch (we call this |
|
63 | 63 | the "trunk"):: |
|
64 | 64 | |
|
65 | 65 | $ bzr branch lp:ipython |
|
66 | 66 | |
|
67 | 67 | Create a working branch |
|
68 | 68 | ----------------------- |
|
69 | 69 | |
|
70 | 70 | When working on IPython, you won't actually make edits directly to the |
|
71 | 71 | :file:`lp:ipython` branch. Instead, you will create a separate branch for your |
|
72 | 72 | changes. For now, let's assume you want to do your work in a branch named |
|
73 | 73 | "ipython-mybranch". Create this branch by doing:: |
|
74 | 74 | |
|
75 | 75 | $ bzr branch ipython ipython-mybranch |
|
76 | 76 | |
|
77 | 77 | When you actually create a branch, you will want to give it a name that |
|
78 | 78 | reflects the nature of the work that you will be doing in it, like |
|
79 | 79 | "install-docs-update". |
|
80 | 80 | |
|
81 | 81 | Make edits in your working branch |
|
82 | 82 | --------------------------------- |
|
83 | 83 | |
|
84 | 84 | Now you are ready to actually make edits in your :file:`ipython-mybranch` |
|
85 | 85 | branch. Before doing this, it is helpful to install this branch so you can |
|
86 | 86 | test your changes as you work. This is easiest if you have setuptools |
|
87 | 87 | installed. Then, just do:: |
|
88 | 88 | |
|
89 | 89 | $ cd ipython-mybranch |
|
90 | 90 | $ python setupegg.py develop |
|
91 | 91 | |
|
92 | 92 | Now, make some changes. After a while, you will want to commit your changes. |
|
93 | 93 | This let's Bazaar know that you like the changes you have made and gives you |
|
94 | 94 | an opportunity to keep a nice record of what you have done. This looks like |
|
95 | 95 | this:: |
|
96 | 96 | |
|
97 | 97 | $ ...do work in ipython-mybranch... |
|
98 | 98 | $ bzr commit -m "the commit message goes here" |
|
99 | 99 | |
|
100 | 100 | Please note that since we now don't use an old-style linear ChangeLog (that |
|
101 | 101 | tends to cause problems with distributed version control systems), you should |
|
102 | 102 | ensure that your log messages are reasonably detailed. Use a docstring-like |
|
103 | 103 | approach in the commit messages (including the second line being left |
|
104 | 104 | *blank*):: |
|
105 | 105 | |
|
106 | 106 | Single line summary of changes being committed. |
|
107 | 107 | |
|
108 | 108 | * more details when warranted ... |
|
109 | 109 | * including crediting outside contributors if they sent the |
|
110 | 110 | code/bug/idea! |
|
111 | 111 | |
|
112 | 112 | As you work, you will repeat this edit/commit cycle many times. If you work on |
|
113 | 113 | your branch for a long time, you will also want to get the latest changes from |
|
114 | 114 | the :file:`lp:ipython` branch. This can be done with the following sequence of |
|
115 | 115 | commands:: |
|
116 | 116 | |
|
117 | 117 | $ ls |
|
118 | 118 | ipython |
|
119 | 119 | ipython-mybranch |
|
120 | 120 | |
|
121 | 121 | $ cd ipython |
|
122 | 122 | $ bzr pull |
|
123 | 123 | $ cd ../ipython-mybranch |
|
124 | 124 | $ bzr merge ../ipython |
|
125 | 125 | $ bzr commit -m "Merging changes from trunk" |
|
126 | 126 | |
|
127 | 127 | Along the way, you should also run the IPython test suite. You can do this |
|
128 | 128 | using the :command:`iptest` command (which is basically a customized version of |
|
129 | 129 | :command:`nosetests`):: |
|
130 | 130 | |
|
131 | 131 | $ cd |
|
132 | 132 | $ iptest |
|
133 | 133 | |
|
134 | 134 | The :command:`iptest` command will also pick up and run any tests you have |
|
135 | 135 | written. See :ref:`testing documentation <devel_testing>` for further details |
|
136 | 136 | on the testing system. |
|
137 | 137 | |
|
138 | 138 | |
|
139 | 139 | Post your branch and request a code review |
|
140 | 140 | ------------------------------------------ |
|
141 | 141 | |
|
142 | 142 | Once you are done with your edits, you should post your branch on Launchpad so |
|
143 | 143 | that other IPython developers can review the changes and help you merge your |
|
144 | 144 | changes into the main development branch. To post your branch on Launchpad, |
|
145 | 145 | do:: |
|
146 | 146 | |
|
147 | 147 | $ cd ipython-mybranch |
|
148 | 148 | $ bzr push lp:~yourusername/ipython/ipython-mybranch |
|
149 | 149 | |
|
150 | 150 | Then, go to the `IPython Launchpad site <www.launchpad.net/ipython>`_, and you |
|
151 | 151 | should see your branch under the "Code" tab. If you click on your branch, you |
|
152 | 152 | can provide a short description of the branch as well as mark its status. Most |
|
153 | 153 | importantly, you should click the link that reads "Propose for merging into |
|
154 | 154 | another branch". What does this do? |
|
155 | 155 | |
|
156 | 156 | This let's the other IPython developers know that your branch is ready to be |
|
157 | 157 | reviewed and merged into the main development branch. During this review |
|
158 | 158 | process, other developers will give you feedback and help you get your code |
|
159 | 159 | ready to be merged. What types of things will we be looking for: |
|
160 | 160 | |
|
161 | 161 | * All code is documented. |
|
162 | 162 | * All code has tests. |
|
163 | 163 | * The entire IPython test suite passes. |
|
164 | 164 | |
|
165 | 165 | Once your changes have been reviewed and approved, someone will merge them |
|
166 | 166 | into the main development branch. |
|
167 | 167 | |
|
168 | 168 | |
|
169 | 169 | Some notes for core developers when merging third-party contributions |
|
170 | 170 | ===================================================================== |
|
171 | 171 | |
|
172 | 172 | Core developers, who ultimately merge any approved branch (from themselves, |
|
173 | 173 | another developer, or any third-party contribution) will typically use |
|
174 | 174 | :command:`bzr merge` to merge the branch into the trunk and push it to the |
|
175 | 175 | main Launcphad site. This is a short list of things to keep in mind when doing |
|
176 | 176 | this process, so that the project history is easy to understand in the long |
|
177 | 177 | run, and that generating release notes is as painless and accurate as |
|
178 | 178 | possible. |
|
179 | 179 | |
|
180 | 180 | - When you merge any non-trivial functionality (from one small bug fix to a |
|
181 | 181 | big feature branch), please remember to always edit the :file:`changes.txt` |
|
182 | 182 | file accordingly. This file has one main section for each release, and if |
|
183 | 183 | you edit it as you go, noting what new features, bug fixes or API changes |
|
184 | 184 | you have made, the release notes will be almost finished when they are |
|
185 | 185 | needed later. This is much easier if done when you merge the work, rather |
|
186 | 186 | than weeks or months later by re-reading a massive Bazaar log. |
|
187 | 187 | |
|
188 | 188 | - When big merges are done, the practice of putting a summary commit message |
|
189 | 189 | in the merge is *extremely* useful. It makes this kind of job much nicer, |
|
190 | 190 | because that summary log message can be almost copy/pasted without changes, |
|
191 | 191 | if it was well written, rather than dissecting the next-level messages from |
|
192 | 192 | the individual commits. |
|
193 | 193 | |
|
194 | 194 | - It's important that we remember to always credit who gave us something if |
|
195 | 195 | it's not the committer. In general, we have been fairly good on this front, |
|
196 | 196 | this is just a reminder to keep things up. As a note, if you are ever |
|
197 | 197 | committing something that is completely (or almost so) a third-party |
|
198 | 198 | contribution, do the commit as:: |
|
199 | 199 | |
|
200 | 200 | $ bzr commit --author="Someone Else" |
|
201 | 201 | |
|
202 | 202 | This way it will show that name separately in the log, which makes it even |
|
203 | 203 | easier to spot. Obviously we often rework third party contributions |
|
204 | 204 | extensively, but this is still good to keep in mind for cases when we don't |
|
205 | 205 | touch the code too much. |
|
206 | 206 | |
|
207 | 207 | |
|
208 | 208 | Documentation |
|
209 | 209 | ============= |
|
210 | 210 | |
|
211 | 211 | Standalone documentation |
|
212 | 212 | ------------------------ |
|
213 | 213 | |
|
214 | 214 | All standalone documentation should be written in plain text (``.txt``) files |
|
215 | 215 | using reStructuredText [reStructuredText]_ for markup and formatting. All such |
|
216 | 216 | documentation should be placed in directory :file:`docs/source` of the IPython |
|
217 | 217 | source tree. The documentation in this location will serve as the main source |
|
218 | 218 | for IPython documentation and all existing documentation should be converted |
|
219 | 219 | to this format. |
|
220 | 220 | |
|
221 | 221 | To build the final documentation, we use Sphinx [Sphinx]_. Once you have |
|
222 | 222 | Sphinx installed, you can build the html docs yourself by doing:: |
|
223 | 223 | |
|
224 | 224 | $ cd ipython-mybranch/docs |
|
225 | 225 | $ make html |
|
226 | 226 | |
|
227 | 227 | Docstring format |
|
228 | 228 | ---------------- |
|
229 | 229 | |
|
230 | 230 | Good docstrings are very important. All new code should have docstrings that |
|
231 | 231 | are formatted using reStructuredText for markup and formatting, since it is |
|
232 | 232 | understood by a wide variety of tools. Details about using reStructuredText |
|
233 | 233 | for docstrings can be found `here |
|
234 | 234 | <http://epydoc.sourceforge.net/manual-othermarkup.html>`_. |
|
235 | 235 | |
|
236 | 236 | Additional PEPs of interest regarding documentation of code: |
|
237 | 237 | |
|
238 | 238 | * `Docstring Conventions <http://www.python.org/peps/pep-0257.html>`_ |
|
239 | 239 | * `Docstring Processing System Framework <http://www.python.org/peps/pep-0256.html>`_ |
|
240 | 240 | * `Docutils Design Specification <http://www.python.org/peps/pep-0258.html>`_ |
|
241 | 241 | |
|
242 | 242 | |
|
243 | 243 | Coding conventions |
|
244 | 244 | ================== |
|
245 | 245 | |
|
246 | 246 | General |
|
247 | 247 | ------- |
|
248 | 248 | |
|
249 | 249 | In general, we'll try to follow the standard Python style conventions as |
|
250 | 250 | described here: |
|
251 | 251 | |
|
252 | 252 | * `Style Guide for Python Code <http://www.python.org/peps/pep-0008.html>`_ |
|
253 | 253 | |
|
254 | 254 | |
|
255 | 255 | Other comments: |
|
256 | 256 | |
|
257 | 257 | * In a large file, top level classes and functions should be |
|
258 | 258 | separated by 2-3 lines to make it easier to separate them visually. |
|
259 | 259 | * Use 4 spaces for indentation. |
|
260 | 260 | * Keep the ordering of methods the same in classes that have the same |
|
261 | 261 | methods. This is particularly true for classes that implement an interface. |
|
262 | 262 | |
|
263 | 263 | Naming conventions |
|
264 | 264 | ------------------ |
|
265 | 265 | |
|
266 | 266 | In terms of naming conventions, we'll follow the guidelines from the `Style |
|
267 | 267 | Guide for Python Code`_. |
|
268 | 268 | |
|
269 | 269 | For all new IPython code (and much existing code is being refactored), we'll |
|
270 | 270 | use: |
|
271 | 271 | |
|
272 | 272 | * All ``lowercase`` module names. |
|
273 | 273 | |
|
274 | 274 | * ``CamelCase`` for class names. |
|
275 | 275 | |
|
276 | 276 | * ``lowercase_with_underscores`` for methods, functions, variables and |
|
277 | 277 | attributes. |
|
278 | 278 | |
|
279 | 279 | There are, however, some important exceptions to these rules. In some cases, |
|
280 | 280 | IPython code will interface with packages (Twisted, Wx, Qt) that use other |
|
281 | 281 | conventions. At some level this makes it impossible to adhere to our own |
|
282 | 282 | standards at all times. In particular, when subclassing classes that use other |
|
283 | 283 | naming conventions, you must follow their naming conventions. To deal with |
|
284 | 284 | cases like this, we propose the following policy: |
|
285 | 285 | |
|
286 | 286 | * If you are subclassing a class that uses different conventions, use its |
|
287 | 287 | naming conventions throughout your subclass. Thus, if you are creating a |
|
288 | 288 | Twisted Protocol class, used Twisted's |
|
289 | 289 | ``namingSchemeForMethodsAndAttributes.`` |
|
290 | 290 | |
|
291 | 291 | * All IPython's official interfaces should use our conventions. In some cases |
|
292 | 292 | this will mean that you need to provide shadow names (first implement |
|
293 | 293 | ``fooBar`` and then ``foo_bar = fooBar``). We want to avoid this at all |
|
294 | 294 | costs, but it will probably be necessary at times. But, please use this |
|
295 | 295 | sparingly! |
|
296 | 296 | |
|
297 | 297 | Implementation-specific *private* methods will use |
|
298 | 298 | ``_single_underscore_prefix``. Names with a leading double underscore will |
|
299 | 299 | *only* be used in special cases, as they makes subclassing difficult (such |
|
300 | 300 | names are not easily seen by child classes). |
|
301 | 301 | |
|
302 | 302 | Occasionally some run-in lowercase names are used, but mostly for very short |
|
303 | 303 | names or where we are implementing methods very similar to existing ones in a |
|
304 | 304 | base class (like ``runlines()`` where ``runsource()`` and ``runcode()`` had |
|
305 | 305 | established precedent). |
|
306 | 306 | |
|
307 | 307 | The old IPython codebase has a big mix of classes and modules prefixed with an |
|
308 | 308 | explicit ``IP``. In Python this is mostly unnecessary, redundant and frowned |
|
309 | 309 | upon, as namespaces offer cleaner prefixing. The only case where this approach |
|
310 | 310 | is justified is for classes which are expected to be imported into external |
|
311 | 311 | namespaces and a very generic name (like Shell) is too likely to clash with |
|
312 | 312 | something else. We'll need to revisit this issue as we clean up and refactor |
|
313 | 313 | the code, but in general we should remove as many unnecessary ``IP``/``ip`` |
|
314 | 314 | prefixes as possible. However, if a prefix seems absolutely necessary the more |
|
315 | 315 | specific ``IPY`` or ``ipy`` are preferred. |
|
316 | 316 | |
|
317 | 317 | .. _devel_testing: |
|
318 | 318 | |
|
319 | 319 | Testing system |
|
320 | 320 | ============== |
|
321 | 321 | |
|
322 | 322 | It is extremely important that all code contributed to IPython has tests. |
|
323 | 323 | Tests should be written as unittests, doctests or as entities that the Nose |
|
324 | 324 | [Nose]_ testing package will find. Regardless of how the tests are written, we |
|
325 | 325 | will use Nose for discovering and running the tests. Nose will be required to |
|
326 | 326 | run the IPython test suite, but will not be required to simply use IPython. |
|
327 | 327 | |
|
328 | 328 | Tests of Twisted using code need to follow two additional guidelines: |
|
329 | 329 | |
|
330 | 330 | 1. Twisted using tests should be written by subclassing the :class:`TestCase` |
|
331 | 331 | class that comes with :mod:`twisted.trial.unittest`. |
|
332 | 332 | |
|
333 | 333 | 2. All :class:`Deferred` instances that are created in the test must be |
|
334 | 334 | properly chained and the final one *must* be the return value of the test |
|
335 | 335 | method. |
|
336 | 336 | |
|
337 | 337 | When these two things are done, Nose will be able to run the tests and the |
|
338 | 338 | twisted reactor will be handled correctly. |
|
339 | 339 | |
|
340 | 340 | Each subpackage in IPython should have its own :file:`tests` directory that |
|
341 | 341 | contains all of the tests for that subpackage. This allows each subpackage to |
|
342 | 342 | be self-contained. A good convention to follow is to have a file named |
|
343 | 343 | :file:`test_foo.py` for each module :file:`foo.py` in the package. This makes |
|
344 | 344 | it easy to organize the tests, though like most conventions, it's OK to break |
|
345 | 345 | it if logic and common sense dictate otherwise. |
|
346 | 346 | |
|
347 | 347 | If a subpackage has any dependencies beyond the Python standard library, the |
|
348 | 348 | tests for that subpackage should be skipped if the dependencies are not |
|
349 | 349 | found. This is very important so users don't get tests failing simply because |
|
350 | 350 | they don't have dependencies. We ship a set of decorators in the |
|
351 | 351 | :mod:`IPython.testing` package to tag tests that may be platform-specific or |
|
352 | 352 | otherwise may have restrictions; if the existing ones don't fit your needs, add |
|
353 | 353 | a new decorator in that location so other tests can reuse it. |
|
354 | 354 | |
|
355 | 355 | To run the IPython test suite, use the :command:`iptest` command that is |
|
356 | 356 | installed with IPython (if you are using IPython in-place, without installing |
|
357 | 357 | it, you can find this script in the :file:`scripts` directory):: |
|
358 | 358 | |
|
359 | 359 | $ iptest |
|
360 | 360 | |
|
361 | 361 | This command colects all IPython tests into separate groups, and then calls |
|
362 | 362 | either Nose with the proper options and extensions, or Twisted's |
|
363 | 363 | :command:`trial`. This ensures that tests that need the Twisted reactor |
|
364 | 364 | management facilities execute separate of Nose. If any individual test group |
|
365 | 365 | fails, :command:`iptest` will print what you need to type so you can rerun that |
|
366 | 366 | particular test group alone for debugging. |
|
367 | 367 | |
|
368 | 368 | By default, :command:`iptest` runs the entire IPython test |
|
369 | 369 | suite (skipping tests that may be platform-specific or which depend on tools |
|
370 | 370 | you may not have). But you can also use it to run only one specific test file, |
|
371 | 371 | or a specific test function. For example, this will run only the |
|
372 | 372 | :file:`test_magic` file from the test suite:: |
|
373 | 373 | |
|
374 | 374 | $ iptest IPython.tests.test_magic |
|
375 | 375 | ---------------------------------------------------------------------- |
|
376 | 376 | Ran 10 tests in 0.348s |
|
377 | 377 | |
|
378 | 378 | OK (SKIP=3) |
|
379 | 379 | Deleting object: second_pass |
|
380 | 380 | |
|
381 | 381 | while the ``path:function`` syntax allows you to select a specific function in |
|
382 | 382 | that file to run:: |
|
383 | 383 | |
|
384 | 384 | $ iptest IPython.tests.test_magic:test_obj_del |
|
385 | 385 | ---------------------------------------------------------------------- |
|
386 | 386 | Ran 1 test in 0.204s |
|
387 | 387 | |
|
388 | 388 | OK |
|
389 | 389 | |
|
390 | 390 | Since :command:`iptest` is based on nosetests, you can pass it any regular |
|
391 | 391 | nosetests option. For example, you can use ``--pdb`` or ``--pdb-failures`` to |
|
392 | 392 | automatically activate the interactive Pdb debugger on errors or failures. See |
|
393 | 393 | the nosetests documentation for further details. |
|
394 | 394 | |
|
395 | 395 | |
|
396 | 396 | A few tips for writing tests |
|
397 | 397 | ---------------------------- |
|
398 | 398 | |
|
399 | 399 | You can write tests either as normal test files, using all the conventions that |
|
400 | 400 | Nose recognizes, or as doctests. Note that *all* IPython functions should have |
|
401 | 401 | at least one example that serves as a doctest, whenever technically feasible. |
|
402 | 402 | However, example doctests should only be in the main docstring if they are *a |
|
403 | 403 | good example*, i.e. if they convey useful information about the function. If |
|
404 | 404 | you simply would like to write a test as a doctest, put it in a separate test |
|
405 | 405 | file and write a no-op function whose only purpose is its docstring. |
|
406 | 406 | |
|
407 | 407 | Note, however, that in a file named :file:`test_X`, functions whose only test |
|
408 | 408 | is their docstring (as a doctest) and which have no test functionality of their |
|
409 | 409 | own, should be called *doctest_foo* instead of *test_foo*, otherwise they get |
|
410 | 410 | double-counted (the empty function call is counted as a test, which just |
|
411 | 411 | inflates tests numbers artificially). This restriction does not apply to |
|
412 | 412 | functions in files with other names, due to how Nose discovers tests. |
|
413 | 413 | |
|
414 | 414 | You can use IPython examples in your docstrings. Those can make full use of |
|
415 | 415 | IPython functionality (magics, variable substitution, etc), but be careful to |
|
416 | 416 | keep them generic enough that they run identically on all Operating Systems. |
|
417 | 417 | |
|
418 | 418 | The prompts in your doctests can be either of the plain Python ``>>>`` variety |
|
419 | 419 | or ``In [1]:`` IPython style. Since this is the IPython system, after all, we |
|
420 | 420 | encourage you to use IPython prompts throughout, unless you are illustrating a |
|
421 | 421 | specific aspect of the normal prompts (such as the ``%doctest_mode`` magic). |
|
422 | 422 | |
|
423 | 423 | If a test isn't safe to run inside the main nose process (e.g. because it loads |
|
424 | 424 | a GUI toolkit), consider running it in a subprocess and capturing its output |
|
425 | 425 | for evaluation and test decision later. Here is an example of how to do it, by |
|
426 | 426 | relying on the builtin ``_ip`` object that contains the public IPython api as |
|
427 | 427 | defined in :mod:`IPython.ipapi`:: |
|
428 | 428 | |
|
429 | 429 | def test_obj_del(): |
|
430 | 430 | """Test that object's __del__ methods are called on exit.""" |
|
431 | 431 | test_dir = os.path.dirname(__file__) |
|
432 | 432 | del_file = os.path.join(test_dir,'obj_del.py') |
|
433 | 433 | out = _ip.IP.getoutput('ipython %s' % del_file) |
|
434 | 434 | nt.assert_equals(out,'object A deleted') |
|
435 | 435 | |
|
436 | 436 | |
|
437 | 437 | |
|
438 | 438 | If a doctest contains input whose output you don't want to verify identically |
|
439 | 439 | via doctest (random output, an object id, etc), you can mark a docstring with |
|
440 | 440 | ``#random``. All of these test will have their code executed but no output |
|
441 | 441 | checking will be done:: |
|
442 | 442 | |
|
443 | 443 | >>> 1+3 |
|
444 | 444 | junk goes here... # random |
|
445 | 445 | |
|
446 | 446 | >>> 1+2 |
|
447 | 447 | again, anything goes #random |
|
448 | 448 | if multiline, the random mark is only needed once. |
|
449 | 449 | |
|
450 | 450 | >>> 1+2 |
|
451 | 451 | You can also put the random marker at the end: |
|
452 | 452 | # random |
|
453 | 453 | |
|
454 | 454 | >>> 1+2 |
|
455 | 455 | # random |
|
456 | 456 | .. or at the beginning. |
|
457 | 457 | |
|
458 | 458 | In a case where you want an *entire* docstring to be executed but not verified |
|
459 | 459 | (this only serves to check that the code runs without crashing, so it should be |
|
460 | 460 | used very sparingly), you can put ``# all-random`` in the docstring. |
|
461 | 461 | |
|
462 | 462 | .. _devel_config: |
|
463 | 463 | |
|
464 | 464 | Release checklist |
|
465 | 465 | ================= |
|
466 | 466 | |
|
467 | 467 | Most of the release process is automated by the :file:`release` script in the |
|
468 | 468 | :file:`tools` directory. This is just a handy reminder for the release manager. |
|
469 | 469 | |
|
470 | 470 | #. First, run :file:`build_release`, which does all the file checking and |
|
471 | 471 | building that the real release script will do. This will let you do test |
|
472 | 472 | installations, check that the build procedure runs OK, etc. You may want to |
|
473 | 473 | disable a few things like multi-version RPM building while testing, because |
|
474 | 474 | otherwise the build takes really long. |
|
475 | 475 | |
|
476 | 476 | #. Run the release script, which makes the tar.gz, eggs and Win32 .exe |
|
477 | 477 | installer. It posts them to the site and registers the release with PyPI. |
|
478 | 478 | |
|
479 | 479 | #. Updating the website with announcements and links to the updated |
|
480 | 480 | changes.txt in html form. Remember to put a short note both on the news |
|
481 | 481 | page of the site and on Launcphad. |
|
482 | 482 | |
|
483 | 483 | #. Drafting a short release announcement with i) highlights and ii) a link to |
|
484 | 484 | the html changes.txt. |
|
485 | 485 | |
|
486 | 486 | #. Make sure that the released version of the docs is live on the site. |
|
487 | 487 | |
|
488 | 488 | #. Celebrate! |
|
489 | 489 | |
|
490 | 490 | Porting to 3.0 |
|
491 | 491 | ============== |
|
492 | 492 | |
|
493 | 493 | There are no definite plans for porting of IPython to python 3. The major |
|
494 | 494 | issue is the dependency on twisted framework for the networking/threading |
|
495 | 495 | stuff. It is possible that it the traditional IPython interactive console |
|
496 | 496 | could be ported more easily since it has no such dependency. Here are a few |
|
497 | 497 | things that will need to be considered when doing such a port especially |
|
498 | 498 | if we want to have a codebase that works directly on both 2.x and 3.x. |
|
499 | 499 | |
|
500 | 500 | 1. The syntax for exceptions changed (PEP 3110). The old |
|
501 | 501 | `except exc, var` changed to `except exc as var`. At last |
|
502 | 502 | count there was 78 occurences of this usage in the codebase. This |
|
503 | 503 | is a particularly problematic issue, because it's not easy to |
|
504 | 504 | implement it in a 2.5-compatible way. |
|
505 | 505 | |
|
506 | 506 | Because it is quite difficult to support simultaneously Python 2.5 and 3.x, we |
|
507 | 507 | will likely at some point put out a release that requires strictly 2.6 and |
|
508 | 508 | abandons 2.5 compatibility. This will then allow us to port the code to using |
|
509 | 509 | :func:`print` as a function, `except exc as var` syntax, etc. But as of |
|
510 | 510 | version 0.11 at least, we will retain Python 2.5 compatibility. |
|
511 | 511 | |
|
512 | 512 | |
|
513 | 513 | .. [Bazaar] Bazaar. http://bazaar-vcs.org/ |
|
514 | 514 | .. [Launchpad] Launchpad. http://www.launchpad.net/ipython |
|
515 | 515 | .. [reStructuredText] reStructuredText. http://docutils.sourceforge.net/rst.html |
|
516 | 516 | .. [Sphinx] Sphinx. http://sphinx.pocoo.org/ |
|
517 | 517 | .. [Nose] Nose: a discovery based unittest extension. http://code.google.com/p/python-nose/ |
|
518 |
@@ -1,84 +1,85 b'' | |||
|
1 | 1 | ============================= |
|
2 | 2 | IPython module reorganization |
|
3 | 3 | ============================= |
|
4 | 4 | |
|
5 | 5 | Currently, IPython has many top-level modules that serve many different |
|
6 | 6 | purposes. The lack of organization make it very difficult for developers to |
|
7 | 7 | work on IPython and understand its design. This document contains notes about |
|
8 | 8 | how we will reorganize the modules into sub-packages. |
|
9 | 9 | |
|
10 | 10 | .. warning:: |
|
11 | 11 | |
|
12 | 12 | This effort will possibly break third party packages that use IPython as |
|
13 | 13 | a library or hack on the IPython internals. |
|
14 | 14 | |
|
15 | 15 | .. warning:: |
|
16 | 16 | |
|
17 | 17 | This effort will result in the removal from IPython of certain modules |
|
18 | 18 | that are not used anymore, don't currently work, are unmaintained, etc. |
|
19 | 19 | |
|
20 | 20 | |
|
21 | 21 | Current subpackges |
|
22 | 22 | ================== |
|
23 | 23 | |
|
24 | 24 | IPython currently has the following sub-packages: |
|
25 | 25 | |
|
26 | 26 | * :mod:`IPython.config` |
|
27 | 27 | |
|
28 | 28 | * :mod:`IPython.Extensions` |
|
29 | 29 | |
|
30 | 30 | * :mod:`IPython.external` |
|
31 | 31 | |
|
32 | 32 | * :mod:`IPython.frontend` |
|
33 | 33 | |
|
34 | 34 | * :mod:`IPython.gui` |
|
35 | 35 | |
|
36 | 36 | * :mod:`IPython.kernel` |
|
37 | 37 | |
|
38 | 38 | * :mod:`IPython.testing` |
|
39 | 39 | |
|
40 | 40 | * :mod:`IPython.tests` |
|
41 | 41 | |
|
42 | 42 | * :mod:`IPython.tools` |
|
43 | 43 | |
|
44 | 44 | * :mod:`IPython.UserConfig` |
|
45 | 45 | |
|
46 | 46 | New Subpackages to be created |
|
47 | 47 | ============================= |
|
48 | 48 | |
|
49 | 49 | We propose to create the following new sub-packages: |
|
50 | 50 | |
|
51 | 51 | * :mod:`IPython.core`. This sub-package will contain the core of the IPython |
|
52 | 52 | interpreter, but none of its extended capabilities. |
|
53 | 53 | |
|
54 | 54 | * :mod:`IPython.lib`. IPython has many extended capabilities that are not part |
|
55 | 55 | of the IPython core. These things will go here. |
|
56 | 56 | |
|
57 | 57 | * :mod:`IPython.utils`. This sub-package will contain anything that might |
|
58 | 58 | eventually be found in the Python standard library, like things in |
|
59 | 59 | :mod:`genutils`. Each sub-module in this sub-package should contain |
|
60 | 60 | functions and classes that serve a single purpose. |
|
61 | 61 | |
|
62 | 62 | * :mod:`IPython.deathrow`. This is for code that is untested and/or rotting |
|
63 | 63 | and needs to be removed from IPython. Eventually all this code will either |
|
64 | 64 | i) be revived by someone willing to maintain it with tests and docs and |
|
65 | 65 | re-included into IPython or 2) be removed from IPython proper, but put into |
|
66 | 66 | a separate top-level (not IPython) package that we keep around. No new code |
|
67 | 67 | will be allowed here. |
|
68 | 68 | |
|
69 | 69 | * :mod:`IPython.quarantine`. This is for code that doesn't meet IPython's |
|
70 | 70 | standards, but that we plan on keeping. To be moved out of this sub-package |
|
71 | 71 | a module needs to have a maintainer, tests and documentation. |
|
72 | 72 | |
|
73 | 73 | Procedure |
|
74 | 74 | ========= |
|
75 | 75 | |
|
76 | 76 | 1. Move the file to its new location with its new name. |
|
77 | 77 | 2. Rename all import statements to reflect the change. |
|
78 | 78 | 3. Run PyFlakes on each changes module. |
|
79 | 79 | 4. Add tests/test_imports.py to test it. |
|
80 | 80 | |
|
81 | 81 | Status |
|
82 | 82 | ====== |
|
83 | 83 | |
|
84 | 84 | This branch was merged into trunk in early August of 2009. |
|
85 |
@@ -1,96 +1,94 b'' | |||
|
1 | 1 | .. _roadmap: |
|
2 | 2 | |
|
3 | 3 | =================== |
|
4 | 4 | Development roadmap |
|
5 | 5 | =================== |
|
6 | 6 | |
|
7 | 7 | IPython is an ambitious project that is still under heavy development. |
|
8 | 8 | However, we want IPython to become useful to as many people as possible, as |
|
9 | 9 | quickly as possible. To help us accomplish this, we are laying out a roadmap |
|
10 | 10 | of where we are headed and what needs to happen to get there. Hopefully, this |
|
11 | 11 | will help the IPython developers figure out the best things to work on for |
|
12 | 12 | each upcoming release. |
|
13 | 13 | |
|
14 | 14 | Work targeted to particular releases |
|
15 | 15 | ==================================== |
|
16 | 16 | |
|
17 | 17 | Release 0.11 |
|
18 | 18 | ------------ |
|
19 | 19 | |
|
20 | 20 | * Full module and package reorganization (done). |
|
21 | 21 | |
|
22 | 22 | * Removal of the threaded shells and new implementation of GUI support |
|
23 | 23 | based on ``PyOSInputHook`` (done). |
|
24 | 24 | |
|
25 | 25 | * Refactor the configuration system (done). |
|
26 | 26 | |
|
27 | 27 | * Prepare to refactor IPython's core by creating a new component and |
|
28 | 28 | application system (done). |
|
29 | 29 | |
|
30 | 30 | * Start to refactor IPython's core by turning everything into components |
|
31 | 31 | (started). |
|
32 | 32 | |
|
33 | 33 | Release 0.12 |
|
34 | 34 | ------------ |
|
35 | 35 | |
|
36 | 36 | * Continue to refactor IPython's core by turning everything into components. |
|
37 | 37 | |
|
38 | 38 | |
|
39 | 39 | Major areas of work |
|
40 | 40 | =================== |
|
41 | 41 | |
|
42 | 42 | Refactoring the main IPython core |
|
43 | 43 | --------------------------------- |
|
44 | 44 | |
|
45 | 45 | During the summer of 2009, we began refactoring IPython's core. The main |
|
46 | 46 | thrust in this work was make the IPython core into a set of loosely coupled |
|
47 | 47 | components. The base component class for this is |
|
48 | 48 | :class:`IPython.core.component.Component`. This section outlines the status |
|
49 | 49 | of this work. |
|
50 | 50 | |
|
51 | 51 | Parts of the IPython core that have been turned into components: |
|
52 | 52 | |
|
53 | 53 | * The main :class:`InteractiveShell` class. |
|
54 | 54 | * The aliases (:mod:`IPython.core.aliases`). |
|
55 | 55 | * The display and builtin traps (:mod:`IPython.core.display_trap` and |
|
56 | 56 | :mod:`IPython.core.builtin_trap`). |
|
57 | 57 | * The prefilter machinery (:mod:`IPython.core.prefilter`). |
|
58 | 58 | |
|
59 | 59 | Parts of the IPythoncore that need to be turned into components: |
|
60 | 60 | |
|
61 | 61 | * Magics. |
|
62 | 62 | * Input and output history management. |
|
63 | 63 | * Prompts. |
|
64 | 64 | * Completers. |
|
65 | 65 | * Logging. |
|
66 | 66 | * Exception handling. |
|
67 | 67 | * Anything else. |
|
68 | 68 | |
|
69 | 69 | Process management for :mod:`IPython.kernel` |
|
70 | 70 | -------------------------------------------- |
|
71 | 71 | |
|
72 | 72 | Performance problems |
|
73 | 73 | -------------------- |
|
74 | 74 | |
|
75 | 75 | Currently, we have a number of performance issues in :mod:`IPython.kernel`: |
|
76 | 76 | |
|
77 | 77 | * The controller stores a large amount of state in Python dictionaries. Under |
|
78 | 78 | heavy usage, these dicts with get very large, causing memory usage problems. |
|
79 | 79 | We need to develop more scalable solutions to this problem. This will also |
|
80 | 80 | help the controller to be more fault tolerant. |
|
81 | 81 | |
|
82 | 82 | * We currently don't have a good way of handling large objects in the |
|
83 | 83 | controller. The biggest problem is that because we don't have any way of |
|
84 | 84 | streaming objects, we get lots of temporary copies in the low-level buffers. |
|
85 | 85 | We need to implement a better serialization approach and true streaming |
|
86 | 86 | support. |
|
87 | 87 | |
|
88 | 88 | * The controller currently unpickles and repickles objects. We need to use the |
|
89 | 89 | [push|pull]_serialized methods instead. |
|
90 | 90 | |
|
91 | 91 | * Currently the controller is a bottleneck. The best approach for this is to |
|
92 | 92 | separate the controller itself into multiple processes, one for the core |
|
93 | 93 | controller and one each for the controller interfaces. |
|
94 | 94 | |
|
95 | ||
|
96 |
@@ -1,105 +1,96 b'' | |||
|
1 | 1 | .. _faq: |
|
2 | 2 | |
|
3 | 3 | ======================================== |
|
4 | 4 | Frequently asked questions |
|
5 | 5 | ======================================== |
|
6 | 6 | |
|
7 | 7 | General questions |
|
8 | 8 | ================= |
|
9 | 9 | |
|
10 | 10 | Questions about parallel computing with IPython |
|
11 | 11 | ================================================ |
|
12 | 12 | |
|
13 | 13 | Will IPython speed my Python code up? |
|
14 | 14 | -------------------------------------- |
|
15 | 15 | |
|
16 | 16 | Yes and no. When converting a serial code to run in parallel, there often many |
|
17 | 17 | difficulty questions that need to be answered, such as: |
|
18 | 18 | |
|
19 | 19 | * How should data be decomposed onto the set of processors? |
|
20 | 20 | |
|
21 | 21 | * What are the data movement patterns? |
|
22 | 22 | |
|
23 | 23 | * Can the algorithm be structured to minimize data movement? |
|
24 | 24 | |
|
25 | 25 | * Is dynamic load balancing important? |
|
26 | 26 | |
|
27 | 27 | We can't answer such questions for you. This is the hard (but fun) work of parallel |
|
28 | 28 | computing. But, once you understand these things IPython will make it easier for you to |
|
29 | 29 | implement a good solution quickly. Most importantly, you will be able to use the |
|
30 | 30 | resulting parallel code interactively. |
|
31 | 31 | |
|
32 | 32 | With that said, if your problem is trivial to parallelize, IPython has a number of |
|
33 | 33 | different interfaces that will enable you to parallelize things is almost no time at |
|
34 | 34 | all. A good place to start is the ``map`` method of our :class:`MultiEngineClient`. |
|
35 | 35 | |
|
36 | 36 | What is the best way to use MPI from Python? |
|
37 | 37 | -------------------------------------------- |
|
38 | 38 | |
|
39 | 39 | What about all the other parallel computing packages in Python? |
|
40 | 40 | --------------------------------------------------------------- |
|
41 | 41 | |
|
42 | 42 | Some of the unique characteristic of IPython are: |
|
43 | 43 | |
|
44 | 44 | * IPython is the only architecture that abstracts out the notion of a |
|
45 | 45 | parallel computation in such a way that new models of parallel computing |
|
46 | 46 | can be explored quickly and easily. If you don't like the models we |
|
47 | 47 | provide, you can simply create your own using the capabilities we provide. |
|
48 | 48 | |
|
49 | 49 | * IPython is asynchronous from the ground up (we use `Twisted`_). |
|
50 | 50 | |
|
51 | 51 | * IPython's architecture is designed to avoid subtle problems |
|
52 | 52 | that emerge because of Python's global interpreter lock (GIL). |
|
53 | 53 | |
|
54 | 54 | * While IPython's architecture is designed to support a wide range |
|
55 | 55 | of novel parallel computing models, it is fully interoperable with |
|
56 | 56 | traditional MPI applications. |
|
57 | 57 | |
|
58 | 58 | * IPython has been used and tested extensively on modern supercomputers. |
|
59 | 59 | |
|
60 | 60 | * IPython's networking layers are completely modular. Thus, is |
|
61 | 61 | straightforward to replace our existing network protocols with |
|
62 | 62 | high performance alternatives (ones based upon Myranet/Infiniband). |
|
63 | 63 | |
|
64 | 64 | * IPython is designed from the ground up to support collaborative |
|
65 | 65 | parallel computing. This enables multiple users to actively develop |
|
66 | 66 | and run the *same* parallel computation. |
|
67 | 67 | |
|
68 | 68 | * Interactivity is a central goal for us. While IPython does not have |
|
69 | 69 | to be used interactivly, it can be. |
|
70 | 70 | |
|
71 | 71 | .. _Twisted: http://www.twistedmatrix.com |
|
72 | 72 | |
|
73 | 73 | Why The IPython controller a bottleneck in my parallel calculation? |
|
74 | 74 | ------------------------------------------------------------------- |
|
75 | 75 | |
|
76 | 76 | A golden rule in parallel computing is that you should only move data around if you |
|
77 | 77 | absolutely need to. The main reason that the controller becomes a bottleneck is that |
|
78 | 78 | too much data is being pushed and pulled to and from the engines. If your algorithm |
|
79 | 79 | is structured in this way, you really should think about alternative ways of |
|
80 | 80 | handling the data movement. Here are some ideas: |
|
81 | 81 | |
|
82 | 82 | 1. Have the engines write data to files on the locals disks of the engines. |
|
83 | 83 | |
|
84 | 84 | 2. Have the engines write data to files on a file system that is shared by |
|
85 | 85 | the engines. |
|
86 | 86 | |
|
87 | 87 | 3. Have the engines write data to a database that is shared by the engines. |
|
88 | 88 | |
|
89 | 89 | 4. Simply keep data in the persistent memory of the engines and move the |
|
90 | 90 | computation to the data (rather than the data to the computation). |
|
91 | 91 | |
|
92 | 92 | 5. See if you can pass data directly between engines using MPI. |
|
93 | 93 | |
|
94 | 94 | Isn't Python slow to be used for high-performance parallel computing? |
|
95 | 95 | --------------------------------------------------------------------- |
|
96 | 96 | |
|
97 | ||
|
98 | ||
|
99 | ||
|
100 | ||
|
101 | ||
|
102 | ||
|
103 | ||
|
104 | ||
|
105 |
@@ -1,31 +1,34 b'' | |||
|
1 | 1 | ===================== |
|
2 | 2 | IPython Documentation |
|
3 | 3 | ===================== |
|
4 | 4 | |
|
5 | 5 | .. htmlonly:: |
|
6 | 6 | |
|
7 | 7 | :Release: |release| |
|
8 | 8 | :Date: |today| |
|
9 | 9 | |
|
10 | Contents: | |
|
10 | Welcome to the official IPython documentation. This document describes the | |
|
11 | various parts of IPython that are relevant to both users and developers. | |
|
12 | ||
|
13 | Contents | |
|
14 | ======== | |
|
11 | 15 | |
|
12 | 16 | .. toctree:: |
|
13 |
:maxdepth: |
|
|
17 | :maxdepth: 1 | |
|
14 | 18 | |
|
15 | 19 | overview.txt |
|
20 | whatsnew/index.txt | |
|
16 | 21 | install/index.txt |
|
17 | 22 | interactive/index.txt |
|
18 | 23 | parallel/index.txt |
|
19 | 24 | config/index.txt |
|
20 | faq.txt | |
|
21 | history.txt | |
|
22 | changes.txt | |
|
23 | 25 | development/index.txt |
|
24 | 26 | api/index.txt |
|
25 | license_and_copyright.txt | |
|
26 | credits.txt | |
|
27 | faq.txt | |
|
28 | about/index.txt | |
|
27 | 29 | |
|
28 | 30 | .. htmlonly:: |
|
29 | 31 | * :ref:`genindex` |
|
30 | 32 | * :ref:`modindex` |
|
31 | 33 | * :ref:`search` |
|
34 |
@@ -1,10 +1,11 b'' | |||
|
1 | 1 | .. _install_index: |
|
2 | 2 | |
|
3 | 3 | ============ |
|
4 | 4 | Installation |
|
5 | 5 | ============ |
|
6 | 6 | |
|
7 | 7 | .. toctree:: |
|
8 | 8 | :maxdepth: 2 |
|
9 | 9 | |
|
10 | 10 | install.txt |
|
11 |
@@ -1,293 +1,294 b'' | |||
|
1 | 1 | Overview |
|
2 | 2 | ======== |
|
3 | 3 | |
|
4 | 4 | This document describes the steps required to install IPython. IPython is |
|
5 | 5 | organized into a number of subpackages, each of which has its own dependencies. |
|
6 | 6 | All of the subpackages come with IPython, so you don't need to download and |
|
7 | 7 | install them separately. However, to use a given subpackage, you will need to |
|
8 | 8 | install all of its dependencies. |
|
9 | 9 | |
|
10 | 10 | |
|
11 | 11 | Please let us know if you have problems installing IPython or any of its |
|
12 | 12 | dependencies. Officially, IPython requires Python version 2.5 or 2.6. We |
|
13 | 13 | have *not* yet started to port IPython to Python 3.0. |
|
14 | 14 | |
|
15 | 15 | .. warning:: |
|
16 | 16 | |
|
17 | 17 | Officially, IPython supports Python versions 2.5 and 2.6. |
|
18 | 18 | |
|
19 | 19 | IPython 0.10 has only been well tested with Python 2.5 and 2.6. Parts of |
|
20 | 20 | it may work with Python 2.4, but we do not officially support Python 2.4 |
|
21 | 21 | anymore. If you need to use 2.4, you can still run IPython 0.9. |
|
22 | 22 | |
|
23 | 23 | Some of the installation approaches use the :mod:`setuptools` package and its |
|
24 | 24 | :command:`easy_install` command line program. In many scenarios, this provides |
|
25 | 25 | the most simple method of installing IPython and its dependencies. It is not |
|
26 | 26 | required though. More information about :mod:`setuptools` can be found on its |
|
27 | 27 | website. |
|
28 | 28 | |
|
29 | 29 | More general information about installing Python packages can be found in |
|
30 | 30 | Python's documentation at http://www.python.org/doc/. |
|
31 | 31 | |
|
32 | 32 | Quickstart |
|
33 | 33 | ========== |
|
34 | 34 | |
|
35 | 35 | If you have :mod:`setuptools` installed and you are on OS X or Linux (not |
|
36 | 36 | Windows), the following will download and install IPython *and* the main |
|
37 | 37 | optional dependencies:: |
|
38 | 38 | |
|
39 | 39 | $ easy_install ipython[kernel,security,test] |
|
40 | 40 | |
|
41 | 41 | This will get Twisted, zope.interface and Foolscap, which are needed for |
|
42 | 42 | IPython's parallel computing features as well as the nose package, which will |
|
43 | 43 | enable you to run IPython's test suite. To run IPython's test suite, use the |
|
44 | 44 | :command:`iptest` command:: |
|
45 | 45 | |
|
46 | 46 | $ iptest |
|
47 | 47 | |
|
48 | 48 | Read on for more specific details and instructions for Windows. |
|
49 | 49 | |
|
50 | 50 | Installing IPython itself |
|
51 | 51 | ========================= |
|
52 | 52 | |
|
53 | 53 | Given a properly built Python, the basic interactive IPython shell will work |
|
54 | 54 | with no external dependencies. However, some Python distributions |
|
55 | 55 | (particularly on Windows and OS X), don't come with a working :mod:`readline` |
|
56 | 56 | module. The IPython shell will work without :mod:`readline`, but will lack |
|
57 | 57 | many features that users depend on, such as tab completion and command line |
|
58 | 58 | editing. See below for details of how to make sure you have a working |
|
59 | 59 | :mod:`readline`. |
|
60 | 60 | |
|
61 | 61 | Installation using easy_install |
|
62 | 62 | ------------------------------- |
|
63 | 63 | |
|
64 | 64 | If you have :mod:`setuptools` installed, the easiest way of getting IPython is |
|
65 | 65 | to simple use :command:`easy_install`:: |
|
66 | 66 | |
|
67 | 67 | $ easy_install ipython |
|
68 | 68 | |
|
69 | 69 | That's it. |
|
70 | 70 | |
|
71 | 71 | Installation from source |
|
72 | 72 | ------------------------ |
|
73 | 73 | |
|
74 | 74 | If you don't want to use :command:`easy_install`, or don't have it installed, |
|
75 | 75 | just grab the latest stable build of IPython from `here |
|
76 | 76 | <http://ipython.scipy.org/dist/>`_. Then do the following:: |
|
77 | 77 | |
|
78 | 78 | $ tar -xzf ipython.tar.gz |
|
79 | 79 | $ cd ipython |
|
80 | 80 | $ python setup.py install |
|
81 | 81 | |
|
82 | 82 | If you are installing to a location (like ``/usr/local``) that requires higher |
|
83 | 83 | permissions, you may need to run the last command with :command:`sudo`. |
|
84 | 84 | |
|
85 | 85 | Windows |
|
86 | 86 | ------- |
|
87 | 87 | |
|
88 | 88 | There are a few caveats for Windows users. The main issue is that a basic |
|
89 | 89 | ``python setup.py install`` approach won't create ``.bat`` file or Start Menu |
|
90 | 90 | shortcuts, which most users want. To get an installation with these, you can |
|
91 | 91 | use any of the following alternatives: |
|
92 | 92 | |
|
93 | 93 | 1. Install using :command:`easy_install`. |
|
94 | 94 | |
|
95 | 95 | 2. Install using our binary ``.exe`` Windows installer, which can be found at |
|
96 | 96 | `here <http://ipython.scipy.org/dist/>`_ |
|
97 | 97 | |
|
98 | 98 | 3. Install from source, but using :mod:`setuptools` (``python setupegg.py |
|
99 | 99 | install``). |
|
100 | 100 | |
|
101 | 101 | IPython by default runs in a termninal window, but the normal terminal |
|
102 | 102 | application supplied by Microsoft Windows is very primitive. You may want to |
|
103 | 103 | download the excellent and free Console_ application instead, which is a far |
|
104 | 104 | superior tool. You can even configure Console to give you by default an |
|
105 | 105 | IPython tab, which is very convenient to create new IPython sessions directly |
|
106 | 106 | from the working terminal. |
|
107 | 107 | |
|
108 | 108 | .. _Console: http://sourceforge.net/projects/console |
|
109 | 109 | |
|
110 | 110 | |
|
111 | 111 | Installing the development version |
|
112 | 112 | ---------------------------------- |
|
113 | 113 | |
|
114 | 114 | It is also possible to install the development version of IPython from our |
|
115 | 115 | `Bazaar <http://bazaar-vcs.org/>`_ source code repository. To do this you will |
|
116 | 116 | need to have Bazaar installed on your system. Then just do:: |
|
117 | 117 | |
|
118 | 118 | $ bzr branch lp:ipython |
|
119 | 119 | $ cd ipython |
|
120 | 120 | $ python setup.py install |
|
121 | 121 | |
|
122 | 122 | Again, this last step on Windows won't create ``.bat`` files or Start Menu |
|
123 | 123 | shortcuts, so you will have to use one of the other approaches listed above. |
|
124 | 124 | |
|
125 | 125 | Some users want to be able to follow the development branch as it changes. If |
|
126 | 126 | you have :mod:`setuptools` installed, this is easy. Simply replace the last |
|
127 | 127 | step by:: |
|
128 | 128 | |
|
129 | 129 | $ python setupegg.py develop |
|
130 | 130 | |
|
131 | 131 | This creates links in the right places and installs the command line script to |
|
132 | 132 | the appropriate places. Then, if you want to update your IPython at any time, |
|
133 | 133 | just do:: |
|
134 | 134 | |
|
135 | 135 | $ bzr pull |
|
136 | 136 | |
|
137 | 137 | Basic optional dependencies |
|
138 | 138 | =========================== |
|
139 | 139 | |
|
140 | 140 | There are a number of basic optional dependencies that most users will want to |
|
141 | 141 | get. These are: |
|
142 | 142 | |
|
143 | 143 | * readline (for command line editing, tab completion, etc.) |
|
144 | 144 | * nose (to run the IPython test suite) |
|
145 | 145 | * pexpect (to use things like irunner) |
|
146 | 146 | |
|
147 | 147 | If you are comfortable installing these things yourself, have at it, otherwise |
|
148 | 148 | read on for more details. |
|
149 | 149 | |
|
150 | 150 | readline |
|
151 | 151 | -------- |
|
152 | 152 | |
|
153 | 153 | In principle, all Python distributions should come with a working |
|
154 | 154 | :mod:`readline` module. But, reality is not quite that simple. There are two |
|
155 | 155 | common situations where you won't have a working :mod:`readline` module: |
|
156 | 156 | |
|
157 | 157 | * If you are using the built-in Python on Mac OS X. |
|
158 | 158 | |
|
159 | 159 | * If you are running Windows, which doesn't have a :mod:`readline` module. |
|
160 | 160 | |
|
161 | 161 | On OS X, the built-in Python doesn't not have :mod:`readline` because of |
|
162 | 162 | license issues. Starting with OS X 10.5 (Leopard), Apple's built-in Python has |
|
163 | 163 | a BSD-licensed not-quite-compatible readline replacement. As of IPython 0.9, |
|
164 | 164 | many of the issues related to the differences between readline and libedit have |
|
165 | 165 | been resolved. For many users, libedit may be sufficient. |
|
166 | 166 | |
|
167 | 167 | Most users on OS X will want to get the full :mod:`readline` module. To get a |
|
168 | 168 | working :mod:`readline` module, just do (with :mod:`setuptools` installed):: |
|
169 | 169 | |
|
170 | 170 | $ easy_install readline |
|
171 | 171 | |
|
172 | 172 | .. note: |
|
173 | 173 | |
|
174 | 174 | Other Python distributions on OS X (such as fink, MacPorts and the |
|
175 | 175 | official python.org binaries) already have readline installed so |
|
176 | 176 | you don't have to do this step. |
|
177 | 177 | |
|
178 | 178 | If needed, the readline egg can be build and installed from source (see the |
|
179 | 179 | wiki page at http://ipython.scipy.org/moin/InstallationOSXLeopard). |
|
180 | 180 | |
|
181 | 181 | On Windows, you will need the PyReadline module. PyReadline is a separate, |
|
182 | 182 | Windows only implementation of readline that uses native Windows calls through |
|
183 | 183 | :mod:`ctypes`. The easiest way of installing PyReadline is you use the binary |
|
184 | 184 | installer available `here <http://ipython.scipy.org/dist/>`_. The :mod:`ctypes` |
|
185 | 185 | module, which comes with Python 2.5 and greater, is required by PyReadline. It |
|
186 | 186 | is available for Python 2.4 at http://python.net/crew/theller/ctypes. |
|
187 | 187 | |
|
188 | 188 | nose |
|
189 | 189 | ---- |
|
190 | 190 | |
|
191 | 191 | To run the IPython test suite you will need the :mod:`nose` package. Nose |
|
192 | 192 | provides a great way of sniffing out and running all of the IPython tests. The |
|
193 | 193 | simplest way of getting nose, is to use :command:`easy_install`:: |
|
194 | 194 | |
|
195 | 195 | $ easy_install nose |
|
196 | 196 | |
|
197 | 197 | Another way of getting this is to do:: |
|
198 | 198 | |
|
199 | 199 | $ easy_install ipython[test] |
|
200 | 200 | |
|
201 | 201 | For more installation options, see the `nose website |
|
202 | 202 | <http://somethingaboutorange.com/mrl/projects/nose/>`_. Once you have nose |
|
203 | 203 | installed, you can run IPython's test suite using the iptest command:: |
|
204 | 204 | |
|
205 | 205 | $ iptest |
|
206 | 206 | |
|
207 | 207 | |
|
208 | 208 | pexpect |
|
209 | 209 | ------- |
|
210 | 210 | |
|
211 | 211 | The `pexpect <http://www.noah.org/wiki/Pexpect>`_ package is used in IPython's |
|
212 | 212 | :command:`irunner` script. On Unix platforms (including OS X), just do:: |
|
213 | 213 | |
|
214 | 214 | $ easy_install pexpect |
|
215 | 215 | |
|
216 | 216 | Windows users are out of luck as pexpect does not run there. |
|
217 | 217 | |
|
218 | 218 | Dependencies for IPython.kernel (parallel computing) |
|
219 | 219 | ==================================================== |
|
220 | 220 | |
|
221 | 221 | The IPython kernel provides a nice architecture for parallel computing. The |
|
222 | 222 | main focus of this architecture is on interactive parallel computing. These |
|
223 | 223 | features require a number of additional packages: |
|
224 | 224 | |
|
225 | 225 | * zope.interface (yep, we use interfaces) |
|
226 | 226 | * Twisted (asynchronous networking framework) |
|
227 | 227 | * Foolscap (a nice, secure network protocol) |
|
228 | 228 | * pyOpenSSL (security for network connections) |
|
229 | 229 | |
|
230 | 230 | On a Unix style platform (including OS X), if you want to use :mod:`setuptools`, you can just do:: |
|
231 | 231 | |
|
232 | 232 | $ easy_install ipython[kernel] # the first three |
|
233 | 233 | $ easy_install ipython[security] # pyOpenSSL |
|
234 | 234 | |
|
235 | 235 | zope.interface and Twisted |
|
236 | 236 | -------------------------- |
|
237 | 237 | |
|
238 | 238 | Twisted [Twisted]_ and zope.interface [ZopeInterface]_ are used for networking |
|
239 | 239 | related things. On Unix style platforms (including OS X), the simplest way of |
|
240 | 240 | getting the these is to use :command:`easy_install`:: |
|
241 | 241 | |
|
242 | 242 | $ easy_install zope.interface |
|
243 | 243 | $ easy_install Twisted |
|
244 | 244 | |
|
245 | 245 | Of course, you can also download the source tarballs from the `Twisted website |
|
246 | 246 | <twistedmatrix.org>`_ and the `zope.interface page at PyPI |
|
247 | 247 | <http://pypi.python.org/pypi/zope.interface>`_ and do the usual ``python |
|
248 | 248 | setup.py install`` if you prefer. |
|
249 | 249 | |
|
250 | 250 | Windows is a bit different. For zope.interface and Twisted, simply get the latest binary ``.exe`` installer from the Twisted website. This installer includes both zope.interface and Twisted and should just work. |
|
251 | 251 | |
|
252 | 252 | Foolscap |
|
253 | 253 | -------- |
|
254 | 254 | |
|
255 | 255 | Foolscap [Foolscap]_ uses Twisted to provide a very nice secure RPC protocol that we use to implement our parallel computing features. |
|
256 | 256 | |
|
257 | 257 | On all platforms a simple:: |
|
258 | 258 | |
|
259 | 259 | $ easy_install foolscap |
|
260 | 260 | |
|
261 | 261 | should work. You can also download the source tarballs from the `Foolscap |
|
262 | 262 | website <http://foolscap.lothar.com/trac>`_ and do ``python setup.py install`` |
|
263 | 263 | if you prefer. |
|
264 | 264 | |
|
265 | 265 | pyOpenSSL |
|
266 | 266 | --------- |
|
267 | 267 | |
|
268 | 268 | IPython requires an older version of pyOpenSSL [pyOpenSSL]_ (0.6 rather than |
|
269 | 269 | the current 0.7). There are a couple of options for getting this: |
|
270 | 270 | |
|
271 | 271 | 1. Most Linux distributions have packages for pyOpenSSL. |
|
272 | 272 | 2. The built-in Python 2.5 on OS X 10.5 already has it installed. |
|
273 | 273 | 3. There are source tarballs on the pyOpenSSL website. On Unix-like |
|
274 | 274 | platforms, these can be built using ``python seutp.py install``. |
|
275 | 275 | 4. There is also a binary ``.exe`` Windows installer on the `pyOpenSSL website <http://pyopenssl.sourceforge.net/>`_. |
|
276 | 276 | |
|
277 | 277 | Dependencies for IPython.frontend (the IPython GUI) |
|
278 | 278 | =================================================== |
|
279 | 279 | |
|
280 | 280 | wxPython |
|
281 | 281 | -------- |
|
282 | 282 | |
|
283 | 283 | Starting with IPython 0.9, IPython has a new IPython.frontend package that has |
|
284 | 284 | a nice wxPython based IPython GUI. As you would expect, this GUI requires |
|
285 | 285 | wxPython. Most Linux distributions have wxPython packages available and the |
|
286 | 286 | built-in Python on OS X comes with wxPython preinstalled. For Windows, a |
|
287 | 287 | binary installer is available on the `wxPython website |
|
288 | 288 | <http://www.wxpython.org/>`_. |
|
289 | 289 | |
|
290 | 290 | .. [Twisted] Twisted matrix. http://twistedmatrix.org |
|
291 | 291 | .. [ZopeInterface] http://pypi.python.org/pypi/zope.interface |
|
292 | 292 | .. [Foolscap] Foolscap network protocol. http://foolscap.lothar.com/trac |
|
293 | 293 | .. [pyOpenSSL] pyOpenSSL. http://pyopenssl.sourceforge.net |
|
294 |
@@ -1,252 +1,253 b'' | |||
|
1 | 1 | ===================== |
|
2 | 2 | IPython extension API |
|
3 | 3 | ===================== |
|
4 | 4 | |
|
5 | 5 | IPython api (defined in IPython/ipapi.py) is the public api that |
|
6 | 6 | should be used for |
|
7 | 7 | |
|
8 | 8 | * Configuration of user preferences (.ipython/ipy_user_conf.py) |
|
9 | 9 | * Creating new profiles (.ipython/ipy_profile_PROFILENAME.py) |
|
10 | 10 | * Writing extensions |
|
11 | 11 | |
|
12 | 12 | Note that by using the extension api for configuration (editing |
|
13 | 13 | ipy_user_conf.py instead of ipythonrc), you get better validity checks |
|
14 | 14 | and get richer functionality - for example, you can import an |
|
15 | 15 | extension and call functions in it to configure it for your purposes. |
|
16 | 16 | |
|
17 | 17 | For an example extension (the 'sh' profile), see |
|
18 | 18 | IPython/extensions/ipy_profile_sh.py. |
|
19 | 19 | |
|
20 | 20 | For the last word on what's available, see the source code of |
|
21 | 21 | IPython/ipapi.py. |
|
22 | 22 | |
|
23 | 23 | |
|
24 | 24 | Getting started |
|
25 | 25 | =============== |
|
26 | 26 | |
|
27 | 27 | If you want to define an extension, create a normal python module that |
|
28 | 28 | can be imported. The module will access IPython functionality through |
|
29 | 29 | the 'ip' object defined below. |
|
30 | 30 | |
|
31 | 31 | If you are creating a new profile (e.g. foobar), name the module as |
|
32 | 32 | 'ipy_profile_foobar.py' and put it in your ~/.ipython directory. Then, |
|
33 | 33 | when you start ipython with the '-p foobar' argument, the module is |
|
34 | 34 | automatically imported on ipython startup. |
|
35 | 35 | |
|
36 | 36 | If you are just doing some per-user configuration, you can either |
|
37 | 37 | |
|
38 | 38 | * Put the commands directly into ipy_user_conf.py. |
|
39 | 39 | |
|
40 | 40 | * Create a new module with your customization code and import *that* |
|
41 | 41 | module in ipy_user_conf.py. This is preferable to the first approach, |
|
42 | 42 | because now you can reuse and distribute your customization code. |
|
43 | 43 | |
|
44 | 44 | Getting a handle to the api |
|
45 | 45 | =========================== |
|
46 | 46 | |
|
47 | 47 | Put this in the start of your module:: |
|
48 | 48 | |
|
49 | 49 | #!python |
|
50 | 50 | import IPython.ipapi |
|
51 | 51 | ip = IPython.ipapi.get() |
|
52 | 52 | |
|
53 | 53 | The 'ip' object will then be used for accessing IPython |
|
54 | 54 | functionality. 'ip' will mean this api object in all the following |
|
55 | 55 | code snippets. The same 'ip' that we just acquired is always |
|
56 | 56 | accessible in interactive IPython sessions by the name _ip - play with |
|
57 | 57 | it like this:: |
|
58 | 58 | |
|
59 | 59 | [~\_ipython]|81> a = 10 |
|
60 | 60 | [~\_ipython]|82> _ip.e |
|
61 | 61 | _ip.ev _ip.ex _ip.expose_magic |
|
62 | 62 | [~\_ipython]|82> _ip.ev('a+13') |
|
63 | 63 | <82> 23 |
|
64 | 64 | |
|
65 | 65 | The _ip object is also used in some examples in this document - it can |
|
66 | 66 | be substituted by 'ip' in non-interactive use. |
|
67 | 67 | |
|
68 | 68 | Changing options |
|
69 | 69 | ================ |
|
70 | 70 | |
|
71 | 71 | The ip object has 'options' attribute that can be used te get/set |
|
72 | 72 | configuration options (just as in the ipythonrc file):: |
|
73 | 73 | |
|
74 | 74 | o = ip.options |
|
75 | 75 | o.autocall = 2 |
|
76 | 76 | o.automagic = 1 |
|
77 | 77 | |
|
78 | 78 | Executing statements in IPython namespace with 'ex' and 'ev' |
|
79 | 79 | ============================================================ |
|
80 | 80 | |
|
81 | 81 | Often, you want to e.g. import some module or define something that |
|
82 | 82 | should be visible in IPython namespace. Use ``ip.ev`` to |
|
83 | 83 | *evaluate* (calculate the value of) expression and ``ip.ex`` to |
|
84 | 84 | '''execute''' a statement:: |
|
85 | 85 | |
|
86 | 86 | # path module will be visible to the interactive session |
|
87 | 87 | ip.ex("from path import path" ) |
|
88 | 88 | |
|
89 | 89 | # define a handy function 'up' that changes the working directory |
|
90 | 90 | |
|
91 | 91 | ip.ex('import os') |
|
92 | 92 | ip.ex("def up(): os.chdir('..')") |
|
93 | 93 | |
|
94 | 94 | |
|
95 | 95 | # _i2 has the input history entry #2, print its value in uppercase. |
|
96 | 96 | print ip.ev('_i2.upper()') |
|
97 | 97 | |
|
98 | 98 | Accessing the IPython namespace |
|
99 | 99 | =============================== |
|
100 | 100 | |
|
101 | 101 | ip.user_ns attribute has a dictionary containing the IPython global |
|
102 | 102 | namespace (the namespace visible in the interactive session). |
|
103 | 103 | |
|
104 | 104 | :: |
|
105 | 105 | |
|
106 | 106 | [~\_ipython]|84> tauno = 555 |
|
107 | 107 | [~\_ipython]|85> _ip.user_ns['tauno'] |
|
108 | 108 | <85> 555 |
|
109 | 109 | |
|
110 | 110 | Defining new magic commands |
|
111 | 111 | =========================== |
|
112 | 112 | |
|
113 | 113 | The following example defines a new magic command, %impall. What the |
|
114 | 114 | command does should be obvious:: |
|
115 | 115 | |
|
116 | 116 | def doimp(self, arg): |
|
117 | 117 | ip = self.api |
|
118 | 118 | ip.ex("import %s; reload(%s); from %s import *" % ( |
|
119 | 119 | arg,arg,arg) |
|
120 | 120 | ) |
|
121 | 121 | |
|
122 | 122 | ip.expose_magic('impall', doimp) |
|
123 | 123 | |
|
124 | 124 | Things to observe in this example: |
|
125 | 125 | |
|
126 | 126 | * Define a function that implements the magic command using the |
|
127 | 127 | ipapi methods defined in this document |
|
128 | 128 | * The first argument of the function is 'self', i.e. the |
|
129 | 129 | interpreter object. It shouldn't be used directly. however. |
|
130 | 130 | The interpreter object is probably *not* going to remain stable |
|
131 | 131 | through IPython versions. |
|
132 | 132 | * Access the ipapi through 'self.api' instead of the global 'ip' object. |
|
133 | 133 | * All the text following the magic command on the command line is |
|
134 | 134 | contained in the second argument |
|
135 | 135 | * Expose the magic by ip.expose_magic() |
|
136 | 136 | |
|
137 | 137 | |
|
138 | 138 | Calling magic functions and system commands |
|
139 | 139 | =========================================== |
|
140 | 140 | |
|
141 | 141 | Use ip.magic() to execute a magic function, and ip.system() to execute |
|
142 | 142 | a system command:: |
|
143 | 143 | |
|
144 | 144 | # go to a bookmark |
|
145 | 145 | ip.magic('%cd -b relfiles') |
|
146 | 146 | |
|
147 | 147 | # execute 'ls -F' system command. Interchangeable with os.system('ls'), really. |
|
148 | 148 | ip.system('ls -F') |
|
149 | 149 | |
|
150 | 150 | Launching IPython instance from normal python code |
|
151 | 151 | ================================================== |
|
152 | 152 | |
|
153 | 153 | Use ipapi.launch_new_instance() with an argument that specifies the |
|
154 | 154 | namespace to use. This can be useful for trivially embedding IPython |
|
155 | 155 | into your program. Here's an example of normal python program test.py |
|
156 | 156 | ('''without''' an existing IPython session) that launches an IPython |
|
157 | 157 | interpreter and regains control when the interpreter is exited:: |
|
158 | 158 | |
|
159 | 159 | [ipython]|1> cat test.py |
|
160 | 160 | my_ns = dict( |
|
161 | 161 | kissa = 15, |
|
162 | 162 | koira = 16) |
|
163 | 163 | import IPython.ipapi |
|
164 | 164 | print "launching IPython instance" |
|
165 | 165 | IPython.ipapi.launch_new_instance(my_ns) |
|
166 | 166 | print "Exited IPython instance!" |
|
167 | 167 | print "New vals:",my_ns['kissa'], my_ns['koira'] |
|
168 | 168 | |
|
169 | 169 | And here's what it looks like when run (note how we don't start it |
|
170 | 170 | from an ipython session):: |
|
171 | 171 | |
|
172 | 172 | Q:\ipython>python test.py |
|
173 | 173 | launching IPython instance |
|
174 | 174 | Py 2.5 (r25:51908, Sep 19 2006, 09:52:17) [MSC v.1310 32 bit (Intel)] IPy 0.7.3b3.r1975 |
|
175 | 175 | [ipython]|1> kissa = 444 |
|
176 | 176 | [ipython]|2> koira = 555 |
|
177 | 177 | [ipython]|3> Exit |
|
178 | 178 | Exited IPython instance! |
|
179 | 179 | New vals: 444 555 |
|
180 | 180 | |
|
181 | 181 | Accessing unexposed functionality |
|
182 | 182 | ================================= |
|
183 | 183 | |
|
184 | 184 | There are still many features that are not exposed via the ipapi. If |
|
185 | 185 | you can't avoid using them, you can use the functionality in |
|
186 | 186 | InteractiveShell object (central IPython session class, defined in |
|
187 | 187 | iplib.py) through ip.IP. |
|
188 | 188 | |
|
189 | 189 | For example:: |
|
190 | 190 | |
|
191 | 191 | [~]|7> _ip.IP.expand_aliases('np','myfile.py') |
|
192 | 192 | <7> 'c:/opt/Notepad++/notepad++.exe myfile.py' |
|
193 | 193 | [~]|8> |
|
194 | 194 | |
|
195 | 195 | Still, it's preferable that if you encounter such a feature, contact |
|
196 | 196 | the IPython team and request that the functionality be exposed in a |
|
197 | 197 | future version of IPython. Things not in ipapi are more likely to |
|
198 | 198 | change over time. |
|
199 | 199 | |
|
200 | 200 | Provided extensions |
|
201 | 201 | =================== |
|
202 | 202 | |
|
203 | 203 | You can see the list of available extensions (and profiles) by doing |
|
204 | 204 | ``import ipy_<TAB>``. Some extensions don't have the ``ipy_`` prefix in |
|
205 | 205 | module name, so you may need to see the contents of IPython/extensions |
|
206 | 206 | folder to see what's available. |
|
207 | 207 | |
|
208 | 208 | You can see a brief documentation of an extension by looking at the |
|
209 | 209 | module docstring:: |
|
210 | 210 | |
|
211 | 211 | [c:p/ipython_main]|190> import ipy_fsops |
|
212 | 212 | [c:p/ipython_main]|191> ipy_fsops? |
|
213 | 213 | |
|
214 | 214 | ... |
|
215 | 215 | |
|
216 | 216 | Docstring: |
|
217 | 217 | File system operations |
|
218 | 218 | |
|
219 | 219 | Contains: Simple variants of normal unix shell commands (icp, imv, irm, |
|
220 | 220 | imkdir, igrep). |
|
221 | 221 | |
|
222 | 222 | You can also install your own extensions - the recommended way is to |
|
223 | 223 | just copy the module to ~/.ipython. Extensions are typically enabled |
|
224 | 224 | by just importing them (e.g. in ipy_user_conf.py), but some extensions |
|
225 | 225 | require additional steps, for example:: |
|
226 | 226 | |
|
227 | 227 | [c:p]|192> import ipy_traits_completer |
|
228 | 228 | [c:p]|193> ipy_traits_completer.activate() |
|
229 | 229 | |
|
230 | 230 | Note that extensions, even if provided in the stock IPython |
|
231 | 231 | installation, are not guaranteed to have the same requirements as the |
|
232 | 232 | rest of IPython - an extension may require external libraries or a |
|
233 | 233 | newer version of Python than what IPython officially requires. An |
|
234 | 234 | extension may also be under a more restrictive license than IPython |
|
235 | 235 | (e.g. ipy_bzr is under GPL). |
|
236 | 236 | |
|
237 | 237 | Just for reference, the list of bundled extensions at the time of |
|
238 | 238 | writing is below: |
|
239 | 239 | |
|
240 | 240 | astyle.py clearcmd.py envpersist.py ext_rescapture.py ibrowse.py |
|
241 | 241 | igrid.py InterpreterExec.py InterpreterPasteInput.py ipipe.py |
|
242 | 242 | ipy_app_completers.py ipy_autoreload.py ipy_bzr.py ipy_completers.py |
|
243 | 243 | ipy_constants.py ipy_defaults.py ipy_editors.py ipy_exportdb.py |
|
244 | 244 | ipy_extutil.py ipy_fsops.py ipy_gnuglobal.py ipy_kitcfg.py |
|
245 | 245 | ipy_legacy.py ipy_leo.py ipy_p4.py ipy_profile_doctest.py |
|
246 | 246 | ipy_profile_none.py ipy_profile_scipy.py ipy_profile_sh.py |
|
247 | 247 | ipy_profile_zope.py ipy_pydb.py ipy_rehashdir.py ipy_render.py |
|
248 | 248 | ipy_server.py ipy_signals.py ipy_stock_completers.py |
|
249 | 249 | ipy_system_conf.py ipy_traits_completer.py ipy_vimserver.py |
|
250 | 250 | ipy_which.py ipy_workdir.py jobctrl.py ledit.py numeric_formats.py |
|
251 | 251 | PhysicalQInput.py PhysicalQInteractive.py pickleshare.py |
|
252 | pspersistence.py win32clip.py __init__.py No newline at end of file | |
|
252 | pspersistence.py win32clip.py __init__.py | |
|
253 |
@@ -1,11 +1,12 b'' | |||
|
1 | 1 | ================================== |
|
2 | 2 | Using IPython for interactive work |
|
3 | 3 | ================================== |
|
4 | 4 | |
|
5 | 5 | .. toctree:: |
|
6 | 6 | :maxdepth: 2 |
|
7 | 7 | |
|
8 | 8 | tutorial.txt |
|
9 | 9 | reference.txt |
|
10 | 10 | shell.txt |
|
11 | 11 | extension_api.txt |
|
12 |
@@ -1,284 +1,285 b'' | |||
|
1 | 1 | .. _ipython_as_shell: |
|
2 | 2 | |
|
3 | 3 | ========================= |
|
4 | 4 | IPython as a system shell |
|
5 | 5 | ========================= |
|
6 | 6 | |
|
7 | 7 | Overview |
|
8 | 8 | ======== |
|
9 | 9 | |
|
10 | 10 | The 'sh' profile optimizes IPython for system shell usage. Apart from |
|
11 | 11 | certain job control functionality that is present in unix (ctrl+z does |
|
12 | 12 | "suspend"), the sh profile should provide you with most of the |
|
13 | 13 | functionality you use daily in system shell, and more. Invoke IPython |
|
14 | 14 | in 'sh' profile by doing 'ipython -p sh', or (in win32) by launching |
|
15 | 15 | the "pysh" shortcut in start menu. |
|
16 | 16 | |
|
17 | 17 | If you want to use the features of sh profile as your defaults (which |
|
18 | 18 | might be a good idea if you use other profiles a lot of the time but |
|
19 | 19 | still want the convenience of sh profile), add ``import ipy_profile_sh`` |
|
20 | 20 | to your ~/.ipython/ipy_user_conf.py. |
|
21 | 21 | |
|
22 | 22 | The 'sh' profile is different from the default profile in that: |
|
23 | 23 | |
|
24 | 24 | * Prompt shows the current directory |
|
25 | 25 | * Spacing between prompts and input is more compact (no padding with |
|
26 | 26 | empty lines). The startup banner is more compact as well. |
|
27 | 27 | * System commands are directly available (in alias table) without |
|
28 | 28 | requesting %rehashx - however, if you install new programs along |
|
29 | 29 | your PATH, you might want to run %rehashx to update the persistent |
|
30 | 30 | alias table |
|
31 | 31 | * Macros are stored in raw format by default. That is, instead of |
|
32 | 32 | '_ip.system("cat foo"), the macro will contain text 'cat foo') |
|
33 | 33 | * Autocall is in full mode |
|
34 | 34 | * Calling "up" does "cd .." |
|
35 | 35 | |
|
36 | 36 | The 'sh' profile is different from the now-obsolete (and unavailable) |
|
37 | 37 | 'pysh' profile in that: |
|
38 | 38 | |
|
39 | 39 | * '$$var = command' and '$var = command' syntax is not supported |
|
40 | 40 | * anymore. Use 'var = !command' instead (incidentally, this is |
|
41 | 41 | * available in all IPython profiles). Note that !!command *will* |
|
42 | 42 | * work. |
|
43 | 43 | |
|
44 | 44 | Aliases |
|
45 | 45 | ======= |
|
46 | 46 | |
|
47 | 47 | All of your $PATH has been loaded as IPython aliases, so you should be |
|
48 | 48 | able to type any normal system command and have it executed. See |
|
49 | 49 | %alias? and %unalias? for details on the alias facilities. See also |
|
50 | 50 | %rehashx? for details on the mechanism used to load $PATH. |
|
51 | 51 | |
|
52 | 52 | |
|
53 | 53 | Directory management |
|
54 | 54 | ==================== |
|
55 | 55 | |
|
56 | 56 | Since each command passed by ipython to the underlying system is executed |
|
57 | 57 | in a subshell which exits immediately, you can NOT use !cd to navigate |
|
58 | 58 | the filesystem. |
|
59 | 59 | |
|
60 | 60 | IPython provides its own builtin '%cd' magic command to move in the |
|
61 | 61 | filesystem (the % is not required with automagic on). It also maintains |
|
62 | 62 | a list of visited directories (use %dhist to see it) and allows direct |
|
63 | 63 | switching to any of them. Type 'cd?' for more details. |
|
64 | 64 | |
|
65 | 65 | %pushd, %popd and %dirs are provided for directory stack handling. |
|
66 | 66 | |
|
67 | 67 | |
|
68 | 68 | Enabled extensions |
|
69 | 69 | ================== |
|
70 | 70 | |
|
71 | 71 | Some extensions, listed below, are enabled as default in this profile. |
|
72 | 72 | |
|
73 | 73 | envpersist |
|
74 | 74 | ---------- |
|
75 | 75 | |
|
76 | 76 | %env can be used to "remember" environment variable manipulations. Examples:: |
|
77 | 77 | |
|
78 | 78 | %env - Show all environment variables |
|
79 | 79 | %env VISUAL=jed - set VISUAL to jed |
|
80 | 80 | %env PATH+=;/foo - append ;foo to PATH |
|
81 | 81 | %env PATH+=;/bar - also append ;bar to PATH |
|
82 | 82 | %env PATH-=/wbin; - prepend /wbin; to PATH |
|
83 | 83 | %env -d VISUAL - forget VISUAL persistent val |
|
84 | 84 | %env -p - print all persistent env modifications |
|
85 | 85 | |
|
86 | 86 | ipy_which |
|
87 | 87 | --------- |
|
88 | 88 | |
|
89 | 89 | %which magic command. Like 'which' in unix, but knows about ipython aliases. |
|
90 | 90 | |
|
91 | 91 | Example:: |
|
92 | 92 | |
|
93 | 93 | [C:/ipython]|14> %which st |
|
94 | 94 | st -> start . |
|
95 | 95 | [C:/ipython]|15> %which d |
|
96 | 96 | d -> dir /w /og /on |
|
97 | 97 | [C:/ipython]|16> %which cp |
|
98 | 98 | cp -> cp |
|
99 | 99 | == c:\bin\cp.exe |
|
100 | 100 | c:\bin\cp.exe |
|
101 | 101 | |
|
102 | 102 | ipy_app_completers |
|
103 | 103 | ------------------ |
|
104 | 104 | |
|
105 | 105 | Custom tab completers for some apps like svn, hg, bzr, apt-get. Try 'apt-get install <TAB>' in debian/ubuntu. |
|
106 | 106 | |
|
107 | 107 | ipy_rehashdir |
|
108 | 108 | ------------- |
|
109 | 109 | |
|
110 | 110 | Allows you to add system command aliases for commands that are not along your path. Let's say that you just installed Putty and want to be able to invoke it without adding it to path, you can create the alias for it with rehashdir:: |
|
111 | 111 | |
|
112 | 112 | [~]|22> cd c:/opt/PuTTY/ |
|
113 | 113 | [c:opt/PuTTY]|23> rehashdir . |
|
114 | 114 | <23> ['pageant', 'plink', 'pscp', 'psftp', 'putty', 'puttygen', 'unins000'] |
|
115 | 115 | |
|
116 | 116 | Now, you can execute any of those commams directly:: |
|
117 | 117 | |
|
118 | 118 | [c:opt/PuTTY]|24> cd |
|
119 | 119 | [~]|25> putty |
|
120 | 120 | |
|
121 | 121 | (the putty window opens). |
|
122 | 122 | |
|
123 | 123 | If you want to store the alias so that it will always be available, do '%store putty'. If you want to %store all these aliases persistently, just do it in a for loop:: |
|
124 | 124 | |
|
125 | 125 | [~]|27> for a in _23: |
|
126 | 126 | |..> %store $a |
|
127 | 127 | |..> |
|
128 | 128 | |..> |
|
129 | 129 | Alias stored: pageant (0, 'c:\\opt\\PuTTY\\pageant.exe') |
|
130 | 130 | Alias stored: plink (0, 'c:\\opt\\PuTTY\\plink.exe') |
|
131 | 131 | Alias stored: pscp (0, 'c:\\opt\\PuTTY\\pscp.exe') |
|
132 | 132 | Alias stored: psftp (0, 'c:\\opt\\PuTTY\\psftp.exe') |
|
133 | 133 | ... |
|
134 | 134 | |
|
135 | 135 | mglob |
|
136 | 136 | ----- |
|
137 | 137 | |
|
138 | 138 | Provide the magic function %mglob, which makes it easier (than the 'find' command) to collect (possibly recursive) file lists. Examples:: |
|
139 | 139 | |
|
140 | 140 | [c:/ipython]|9> mglob *.py |
|
141 | 141 | [c:/ipython]|10> mglob *.py rec:*.txt |
|
142 | 142 | [c:/ipython]|19> workfiles = %mglob !.svn/ !.hg/ !*_Data/ !*.bak rec:. |
|
143 | 143 | |
|
144 | 144 | Note that the first 2 calls will put the file list in result history (_, _9, _10), and the last one will assign it to 'workfiles'. |
|
145 | 145 | |
|
146 | 146 | |
|
147 | 147 | Prompt customization |
|
148 | 148 | ==================== |
|
149 | 149 | |
|
150 | 150 | The sh profile uses the following prompt configurations:: |
|
151 | 151 | |
|
152 | 152 | o.prompt_in1= r'\C_LightBlue[\C_LightCyan\Y2\C_LightBlue]\C_Green|\#>' |
|
153 | 153 | o.prompt_in2= r'\C_Green|\C_LightGreen\D\C_Green>' |
|
154 | 154 | |
|
155 | 155 | You can change the prompt configuration to your liking by editing |
|
156 | 156 | ipy_user_conf.py. |
|
157 | 157 | |
|
158 | 158 | String lists |
|
159 | 159 | ============ |
|
160 | 160 | |
|
161 | 161 | String lists (IPython.genutils.SList) are handy way to process output |
|
162 | 162 | from system commands. They are produced by ``var = !cmd`` syntax. |
|
163 | 163 | |
|
164 | 164 | First, we acquire the output of 'ls -l':: |
|
165 | 165 | |
|
166 | 166 | [Q:doc/examples]|2> lines = !ls -l |
|
167 | 167 | == |
|
168 | 168 | ['total 23', |
|
169 | 169 | '-rw-rw-rw- 1 ville None 1163 Sep 30 2006 example-demo.py', |
|
170 | 170 | '-rw-rw-rw- 1 ville None 1927 Sep 30 2006 example-embed-short.py', |
|
171 | 171 | '-rwxrwxrwx 1 ville None 4606 Sep 1 17:15 example-embed.py', |
|
172 | 172 | '-rwxrwxrwx 1 ville None 1017 Sep 30 2006 example-gnuplot.py', |
|
173 | 173 | '-rwxrwxrwx 1 ville None 339 Jun 11 18:01 extension.py', |
|
174 | 174 | '-rwxrwxrwx 1 ville None 113 Dec 20 2006 seteditor.py', |
|
175 | 175 | '-rwxrwxrwx 1 ville None 245 Dec 12 2006 seteditor.pyc'] |
|
176 | 176 | |
|
177 | 177 | Now, let's take a look at the contents of 'lines' (the first number is |
|
178 | 178 | the list element number):: |
|
179 | 179 | |
|
180 | 180 | [Q:doc/examples]|3> lines |
|
181 | 181 | <3> SList (.p, .n, .l, .s, .grep(), .fields() available). Value: |
|
182 | 182 | |
|
183 | 183 | 0: total 23 |
|
184 | 184 | 1: -rw-rw-rw- 1 ville None 1163 Sep 30 2006 example-demo.py |
|
185 | 185 | 2: -rw-rw-rw- 1 ville None 1927 Sep 30 2006 example-embed-short.py |
|
186 | 186 | 3: -rwxrwxrwx 1 ville None 4606 Sep 1 17:15 example-embed.py |
|
187 | 187 | 4: -rwxrwxrwx 1 ville None 1017 Sep 30 2006 example-gnuplot.py |
|
188 | 188 | 5: -rwxrwxrwx 1 ville None 339 Jun 11 18:01 extension.py |
|
189 | 189 | 6: -rwxrwxrwx 1 ville None 113 Dec 20 2006 seteditor.py |
|
190 | 190 | 7: -rwxrwxrwx 1 ville None 245 Dec 12 2006 seteditor.pyc |
|
191 | 191 | |
|
192 | 192 | Now, let's filter out the 'embed' lines:: |
|
193 | 193 | |
|
194 | 194 | [Q:doc/examples]|4> l2 = lines.grep('embed',prune=1) |
|
195 | 195 | [Q:doc/examples]|5> l2 |
|
196 | 196 | <5> SList (.p, .n, .l, .s, .grep(), .fields() available). Value: |
|
197 | 197 | |
|
198 | 198 | 0: total 23 |
|
199 | 199 | 1: -rw-rw-rw- 1 ville None 1163 Sep 30 2006 example-demo.py |
|
200 | 200 | 2: -rwxrwxrwx 1 ville None 1017 Sep 30 2006 example-gnuplot.py |
|
201 | 201 | 3: -rwxrwxrwx 1 ville None 339 Jun 11 18:01 extension.py |
|
202 | 202 | 4: -rwxrwxrwx 1 ville None 113 Dec 20 2006 seteditor.py |
|
203 | 203 | 5: -rwxrwxrwx 1 ville None 245 Dec 12 2006 seteditor.pyc |
|
204 | 204 | |
|
205 | 205 | Now, we want strings having just file names and permissions:: |
|
206 | 206 | |
|
207 | 207 | [Q:doc/examples]|6> l2.fields(8,0) |
|
208 | 208 | <6> SList (.p, .n, .l, .s, .grep(), .fields() available). Value: |
|
209 | 209 | |
|
210 | 210 | 0: total |
|
211 | 211 | 1: example-demo.py -rw-rw-rw- |
|
212 | 212 | 2: example-gnuplot.py -rwxrwxrwx |
|
213 | 213 | 3: extension.py -rwxrwxrwx |
|
214 | 214 | 4: seteditor.py -rwxrwxrwx |
|
215 | 215 | 5: seteditor.pyc -rwxrwxrwx |
|
216 | 216 | |
|
217 | 217 | Note how the line with 'total' does not raise IndexError. |
|
218 | 218 | |
|
219 | 219 | If you want to split these (yielding lists), call fields() without |
|
220 | 220 | arguments:: |
|
221 | 221 | |
|
222 | 222 | [Q:doc/examples]|7> _.fields() |
|
223 | 223 | <7> |
|
224 | 224 | [['total'], |
|
225 | 225 | ['example-demo.py', '-rw-rw-rw-'], |
|
226 | 226 | ['example-gnuplot.py', '-rwxrwxrwx'], |
|
227 | 227 | ['extension.py', '-rwxrwxrwx'], |
|
228 | 228 | ['seteditor.py', '-rwxrwxrwx'], |
|
229 | 229 | ['seteditor.pyc', '-rwxrwxrwx']] |
|
230 | 230 | |
|
231 | 231 | If you want to pass these separated with spaces to a command (typical |
|
232 | 232 | for lists if files), use the .s property:: |
|
233 | 233 | |
|
234 | 234 | |
|
235 | 235 | [Q:doc/examples]|13> files = l2.fields(8).s |
|
236 | 236 | [Q:doc/examples]|14> files |
|
237 | 237 | <14> 'example-demo.py example-gnuplot.py extension.py seteditor.py seteditor.pyc' |
|
238 | 238 | [Q:doc/examples]|15> ls $files |
|
239 | 239 | example-demo.py example-gnuplot.py extension.py seteditor.py seteditor.pyc |
|
240 | 240 | |
|
241 | 241 | SLists are inherited from normal python lists, so every list method is |
|
242 | 242 | available:: |
|
243 | 243 | |
|
244 | 244 | [Q:doc/examples]|21> lines.append('hey') |
|
245 | 245 | |
|
246 | 246 | |
|
247 | 247 | Real world example: remove all files outside version control |
|
248 | 248 | ============================================================ |
|
249 | 249 | |
|
250 | 250 | First, capture output of "hg status":: |
|
251 | 251 | |
|
252 | 252 | [Q:/ipython]|28> out = !hg status |
|
253 | 253 | == |
|
254 | 254 | ['M IPython\\extensions\\ipy_kitcfg.py', |
|
255 | 255 | 'M IPython\\extensions\\ipy_rehashdir.py', |
|
256 | 256 | ... |
|
257 | 257 | '? build\\lib\\IPython\\Debugger.py', |
|
258 | 258 | '? build\\lib\\IPython\\extensions\\InterpreterExec.py', |
|
259 | 259 | '? build\\lib\\IPython\\extensions\\InterpreterPasteInput.py', |
|
260 | 260 | ... |
|
261 | 261 | |
|
262 | 262 | (lines starting with ? are not under version control). |
|
263 | 263 | |
|
264 | 264 | :: |
|
265 | 265 | |
|
266 | 266 | [Q:/ipython]|35> junk = out.grep(r'^\?').fields(1) |
|
267 | 267 | [Q:/ipython]|36> junk |
|
268 | 268 | <36> SList (.p, .n, .l, .s, .grep(), .fields() availab |
|
269 | 269 | ... |
|
270 | 270 | 10: build\bdist.win32\winexe\temp\_ctypes.py |
|
271 | 271 | 11: build\bdist.win32\winexe\temp\_hashlib.py |
|
272 | 272 | 12: build\bdist.win32\winexe\temp\_socket.py |
|
273 | 273 | |
|
274 | 274 | Now we can just remove these files by doing 'rm $junk.s'. |
|
275 | 275 | |
|
276 | 276 | The .s, .n, .p properties |
|
277 | 277 | ========================= |
|
278 | 278 | |
|
279 | 279 | The '.s' property returns one string where lines are separated by |
|
280 | 280 | single space (for convenient passing to system commands). The '.n' |
|
281 | 281 | property return one string where the lines are separated by '\n' |
|
282 | 282 | (i.e. the original output of the function). If the items in string |
|
283 | 283 | list are file names, '.p' can be used to get a list of "path" objects |
|
284 | for convenient file manipulation. No newline at end of file | |
|
284 | for convenient file manipulation. | |
|
285 |
@@ -1,315 +1,317 b'' | |||
|
1 | 1 | .. _tutorial: |
|
2 | 2 | |
|
3 | 3 | ====================== |
|
4 | 4 | Quick IPython tutorial |
|
5 | 5 | ====================== |
|
6 | 6 | |
|
7 | 7 | IPython can be used as an improved replacement for the Python prompt, |
|
8 | 8 | and for that you don't really need to read any more of this manual. But |
|
9 | 9 | in this section we'll try to summarize a few tips on how to make the |
|
10 | 10 | most effective use of it for everyday Python development, highlighting |
|
11 | 11 | things you might miss in the rest of the manual (which is getting long). |
|
12 | 12 | We'll give references to parts in the manual which provide more detail |
|
13 | 13 | when appropriate. |
|
14 | 14 | |
|
15 | 15 | The following article by Jeremy Jones provides an introductory tutorial |
|
16 | 16 | about IPython: http://www.onlamp.com/pub/a/python/2005/01/27/ipython.html |
|
17 | 17 | |
|
18 | 18 | Highlights |
|
19 | 19 | ========== |
|
20 | 20 | |
|
21 | 21 | Tab completion |
|
22 | 22 | -------------- |
|
23 | 23 | |
|
24 | 24 | TAB-completion, especially for attributes, is a convenient way to explore the |
|
25 | 25 | structure of any object you're dealing with. Simply type object_name.<TAB> and |
|
26 | 26 | a list of the object's attributes will be printed (see :ref:`the readline |
|
27 | 27 | section <readline>` for more). Tab completion also works on file and directory |
|
28 | 28 | names, which combined with IPython's alias system allows you to do from within |
|
29 | 29 | IPython many of the things you normally would need the system shell for. |
|
30 | 30 | |
|
31 | 31 | Explore your objects |
|
32 | 32 | -------------------- |
|
33 | 33 | |
|
34 | 34 | Typing object_name? will print all sorts of details about any object, |
|
35 | 35 | including docstrings, function definition lines (for call arguments) and |
|
36 | 36 | constructor details for classes. The magic commands %pdoc, %pdef, %psource |
|
37 | 37 | and %pfile will respectively print the docstring, function definition line, |
|
38 | 38 | full source code and the complete file for any object (when they can be |
|
39 | 39 | found). If automagic is on (it is by default), you don't need to type the '%' |
|
40 | 40 | explicitly. See :ref:`this section <dynamic_object_info>` for more. |
|
41 | 41 | |
|
42 | 42 | The `%run` magic command |
|
43 | 43 | ------------------------ |
|
44 | 44 | |
|
45 | 45 | The %run magic command allows you to run any python script and load all of its |
|
46 | 46 | data directly into the interactive namespace. Since the file is re-read from |
|
47 | 47 | disk each time, changes you make to it are reflected immediately (in contrast |
|
48 | 48 | to the behavior of import). I rarely use import for code I am testing, relying |
|
49 | 49 | on %run instead. See :ref:`this section <magic>` for more on this and other |
|
50 | 50 | magic commands, or type the name of any magic command and ? to get details on |
|
51 | 51 | it. See also :ref:`this section <dreload>` for a recursive reload command. %run |
|
52 | 52 | also has special flags for timing the execution of your scripts (-t) and for |
|
53 | 53 | executing them under the control of either Python's pdb debugger (-d) or |
|
54 | 54 | profiler (-p). With all of these, %run can be used as the main tool for |
|
55 | 55 | efficient interactive development of code which you write in your editor of |
|
56 | 56 | choice. |
|
57 | 57 | |
|
58 | 58 | Debug a Python script |
|
59 | 59 | --------------------- |
|
60 | 60 | |
|
61 | 61 | Use the Python debugger, pdb. The %pdb command allows you to toggle on and off |
|
62 | 62 | the automatic invocation of an IPython-enhanced pdb debugger (with coloring, |
|
63 | 63 | tab completion and more) at any uncaught exception. The advantage of this is |
|
64 | 64 | that pdb starts inside the function where the exception occurred, with all data |
|
65 | 65 | still available. You can print variables, see code, execute statements and even |
|
66 | 66 | walk up and down the call stack to track down the true source of the problem |
|
67 | 67 | (which often is many layers in the stack above where the exception gets |
|
68 | 68 | triggered). Running programs with %run and pdb active can be an efficient to |
|
69 | 69 | develop and debug code, in many cases eliminating the need for print statements |
|
70 | 70 | or external debugging tools. I often simply put a 1/0 in a place where I want |
|
71 | 71 | to take a look so that pdb gets called, quickly view whatever variables I need |
|
72 | 72 | to or test various pieces of code and then remove the 1/0. Note also that '%run |
|
73 | 73 | -d' activates pdb and automatically sets initial breakpoints for you to step |
|
74 | 74 | through your code, watch variables, etc. The :ref:`output caching section |
|
75 | 75 | <output_caching>` has more details. |
|
76 | 76 | |
|
77 | 77 | Use the output cache |
|
78 | 78 | -------------------- |
|
79 | 79 | |
|
80 | 80 | All output results are automatically stored in a global dictionary named Out |
|
81 | 81 | and variables named _1, _2, etc. alias them. For example, the result of input |
|
82 | 82 | line 4 is available either as Out[4] or as _4. Additionally, three variables |
|
83 | 83 | named _, __ and ___ are always kept updated with the for the last three |
|
84 | 84 | results. This allows you to recall any previous result and further use it for |
|
85 | 85 | new calculations. See :ref:`the output caching section <output_caching>` for |
|
86 | 86 | more. |
|
87 | 87 | |
|
88 | 88 | Suppress output |
|
89 | 89 | --------------- |
|
90 | 90 | |
|
91 | 91 | Put a ';' at the end of a line to suppress the printing of output. This is |
|
92 | 92 | useful when doing calculations which generate long output you are not |
|
93 | 93 | interested in seeing. The _* variables and the Out[] list do get updated with |
|
94 | 94 | the contents of the output, even if it is not printed. You can thus still |
|
95 | 95 | access the generated results this way for further processing. |
|
96 | 96 | |
|
97 | 97 | Input cache |
|
98 | 98 | ----------- |
|
99 | 99 | |
|
100 | 100 | A similar system exists for caching input. All input is stored in a global |
|
101 | 101 | list called In , so you can re-execute lines 22 through 28 plus line 34 by |
|
102 | 102 | typing 'exec In[22:29]+In[34]' (using Python slicing notation). If you need |
|
103 | 103 | to execute the same set of lines often, you can assign them to a macro with |
|
104 | 104 | the %macro function. See :ref:`here <input_caching>` for more. |
|
105 | 105 | |
|
106 | 106 | Use your input history |
|
107 | 107 | ---------------------- |
|
108 | 108 | |
|
109 | 109 | The %hist command can show you all previous input, without line numbers if |
|
110 | 110 | desired (option -n) so you can directly copy and paste code either back in |
|
111 | 111 | IPython or in a text editor. You can also save all your history by turning on |
|
112 | 112 | logging via %logstart; these logs can later be either reloaded as IPython |
|
113 | 113 | sessions or used as code for your programs. |
|
114 | 114 | |
|
115 | 115 | Define your own system aliases |
|
116 | 116 | ------------------------------ |
|
117 | 117 | |
|
118 | 118 | Even though IPython gives you access to your system shell via the ! prefix, |
|
119 | 119 | it is convenient to have aliases to the system commands you use most often. |
|
120 | 120 | This allows you to work seamlessly from inside IPython with the same commands |
|
121 | 121 | you are used to in your system shell. IPython comes with some pre-defined |
|
122 | 122 | aliases and a complete system for changing directories, both via a stack (see |
|
123 | 123 | %pushd, %popd and %dhist) and via direct %cd. The latter keeps a history of |
|
124 | 124 | visited directories and allows you to go to any previously visited one. |
|
125 | 125 | |
|
126 | 126 | Call system shell commands |
|
127 | 127 | -------------------------- |
|
128 | 128 | |
|
129 | 129 | Use Python to manipulate the results of system commands. The '!!' special |
|
130 | 130 | syntax, and the %sc and %sx magic commands allow you to capture system output |
|
131 | 131 | into Python variables. |
|
132 | 132 | |
|
133 | 133 | Use Python variables when calling the shell |
|
134 | 134 | ------------------------------------------- |
|
135 | 135 | |
|
136 | 136 | Expand python variables when calling the shell (either via '!' and '!!' or via |
|
137 | 137 | aliases) by prepending a $ in front of them. You can also expand complete |
|
138 | 138 | python expressions. See :ref:`our shell section <system_shell_access>` for |
|
139 | 139 | more details. |
|
140 | 140 | |
|
141 | 141 | Use profiles |
|
142 | 142 | ------------ |
|
143 | 143 | |
|
144 | 144 | Use profiles to maintain different configurations (modules to load, function |
|
145 | 145 | definitions, option settings) for particular tasks. You can then have |
|
146 | 146 | customized versions of IPython for specific purposes. :ref:`This section |
|
147 | 147 | <profiles>` has more details. |
|
148 | 148 | |
|
149 | 149 | |
|
150 | 150 | Embed IPython in your programs |
|
151 | 151 | ------------------------------ |
|
152 | 152 | |
|
153 | 153 | A few lines of code are enough to load a complete IPython inside your own |
|
154 | 154 | programs, giving you the ability to work with your data interactively after |
|
155 | 155 | automatic processing has been completed. See :ref:`here <embedding>` for more. |
|
156 | 156 | |
|
157 | 157 | Use the Python profiler |
|
158 | 158 | ----------------------- |
|
159 | 159 | |
|
160 | 160 | When dealing with performance issues, the %run command with a -p option |
|
161 | 161 | allows you to run complete programs under the control of the Python profiler. |
|
162 | 162 | The %prun command does a similar job for single Python expressions (like |
|
163 | 163 | function calls). |
|
164 | 164 | |
|
165 | 165 | Use IPython to present interactive demos |
|
166 | 166 | ---------------------------------------- |
|
167 | 167 | |
|
168 | 168 | Use the IPython.demo.Demo class to load any Python script as an interactive |
|
169 | 169 | demo. With a minimal amount of simple markup, you can control the execution of |
|
170 | 170 | the script, stopping as needed. See :ref:`here <interactive_demos>` for more. |
|
171 | 171 | |
|
172 | 172 | Run doctests |
|
173 | 173 | ------------ |
|
174 | 174 | |
|
175 | 175 | Run your doctests from within IPython for development and debugging. The |
|
176 | 176 | special %doctest_mode command toggles a mode where the prompt, output and |
|
177 | 177 | exceptions display matches as closely as possible that of the default Python |
|
178 | 178 | interpreter. In addition, this mode allows you to directly paste in code that |
|
179 | 179 | contains leading '>>>' prompts, even if they have extra leading whitespace |
|
180 | 180 | (as is common in doctest files). This combined with the '%history -tn' call |
|
181 | 181 | to see your translated history (with these extra prompts removed and no line |
|
182 | 182 | numbers) allows for an easy doctest workflow, where you can go from doctest |
|
183 | 183 | to interactive execution to pasting into valid Python code as needed. |
|
184 | 184 | |
|
185 | 185 | Source code handling tips |
|
186 | 186 | ========================= |
|
187 | 187 | |
|
188 | 188 | IPython is a line-oriented program, without full control of the |
|
189 | 189 | terminal. Therefore, it doesn't support true multiline editing. However, |
|
190 | 190 | it has a number of useful tools to help you in dealing effectively with |
|
191 | 191 | more complex editing. |
|
192 | 192 | |
|
193 | 193 | The %edit command gives a reasonable approximation of multiline editing, |
|
194 | 194 | by invoking your favorite editor on the spot. IPython will execute the |
|
195 | 195 | code you type in there as if it were typed interactively. Type %edit? |
|
196 | 196 | for the full details on the edit command. |
|
197 | 197 | |
|
198 | 198 | If you have typed various commands during a session, which you'd like to |
|
199 | 199 | reuse, IPython provides you with a number of tools. Start by using %hist |
|
200 | 200 | to see your input history, so you can see the line numbers of all input. |
|
201 | 201 | Let us say that you'd like to reuse lines 10 through 20, plus lines 24 |
|
202 | 202 | and 28. All the commands below can operate on these with the syntax:: |
|
203 | 203 | |
|
204 | 204 | %command 10-20 24 28 |
|
205 | 205 | |
|
206 | 206 | where the command given can be: |
|
207 | 207 | |
|
208 | 208 | * %macro <macroname>: this stores the lines into a variable which, |
|
209 | 209 | when called at the prompt, re-executes the input. Macros can be |
|
210 | 210 | edited later using '%edit macroname', and they can be stored |
|
211 | 211 | persistently across sessions with '%store macroname' (the storage |
|
212 | 212 | system is per-profile). The combination of quick macros, |
|
213 | 213 | persistent storage and editing, allows you to easily refine |
|
214 | 214 | quick-and-dirty interactive input into permanent utilities, always |
|
215 | 215 | available both in IPython and as files for general reuse. |
|
216 | 216 | * %edit: this will open a text editor with those lines pre-loaded |
|
217 | 217 | for further modification. It will then execute the resulting |
|
218 | 218 | file's contents as if you had typed it at the prompt. |
|
219 | 219 | * %save <filename>: this saves the lines directly to a named file on |
|
220 | 220 | disk. |
|
221 | 221 | |
|
222 | 222 | While %macro saves input lines into memory for interactive re-execution, |
|
223 | 223 | sometimes you'd like to save your input directly to a file. The %save |
|
224 | 224 | magic does this: its input sytnax is the same as %macro, but it saves |
|
225 | 225 | your input directly to a Python file. Note that the %logstart command |
|
226 | 226 | also saves input, but it logs all input to disk (though you can |
|
227 | 227 | temporarily suspend it and reactivate it with %logoff/%logon); %save |
|
228 | 228 | allows you to select which lines of input you need to save. |
|
229 | 229 | |
|
230 | 230 | |
|
231 | 231 | Lightweight 'version control' |
|
232 | 232 | ============================= |
|
233 | 233 | |
|
234 | 234 | When you call %edit with no arguments, IPython opens an empty editor |
|
235 | 235 | with a temporary file, and it returns the contents of your editing |
|
236 | 236 | session as a string variable. Thanks to IPython's output caching |
|
237 | 237 | mechanism, this is automatically stored:: |
|
238 | 238 | |
|
239 | 239 | In [1]: %edit |
|
240 | 240 | |
|
241 | 241 | IPython will make a temporary file named: /tmp/ipython_edit_yR-HCN.py |
|
242 | 242 | |
|
243 | 243 | Editing... done. Executing edited code... |
|
244 | 244 | |
|
245 | 245 | hello - this is a temporary file |
|
246 | 246 | |
|
247 | 247 | Out[1]: "print 'hello - this is a temporary file'\n" |
|
248 | 248 | |
|
249 | 249 | Now, if you call '%edit -p', IPython tries to open an editor with the |
|
250 | 250 | same data as the last time you used %edit. So if you haven't used %edit |
|
251 | 251 | in the meantime, this same contents will reopen; however, it will be |
|
252 | 252 | done in a new file. This means that if you make changes and you later |
|
253 | 253 | want to find an old version, you can always retrieve it by using its |
|
254 | 254 | output number, via '%edit _NN', where NN is the number of the output |
|
255 | 255 | prompt. |
|
256 | 256 | |
|
257 | 257 | Continuing with the example above, this should illustrate this idea:: |
|
258 | 258 | |
|
259 | 259 | In [2]: edit -p |
|
260 | 260 | |
|
261 | 261 | IPython will make a temporary file named: /tmp/ipython_edit_nA09Qk.py |
|
262 | 262 | |
|
263 | 263 | Editing... done. Executing edited code... |
|
264 | 264 | |
|
265 | 265 | hello - now I made some changes |
|
266 | 266 | |
|
267 | 267 | Out[2]: "print 'hello - now I made some changes'\n" |
|
268 | 268 | |
|
269 | 269 | In [3]: edit _1 |
|
270 | 270 | |
|
271 | 271 | IPython will make a temporary file named: /tmp/ipython_edit_gy6-zD.py |
|
272 | 272 | |
|
273 | 273 | Editing... done. Executing edited code... |
|
274 | 274 | |
|
275 | 275 | hello - this is a temporary file |
|
276 | 276 | |
|
277 | 277 | IPython version control at work :) |
|
278 | 278 | |
|
279 | 279 | Out[3]: "print 'hello - this is a temporary file'\nprint 'IPython version control at work :)'\n" |
|
280 | 280 | |
|
281 | 281 | |
|
282 | 282 | This section was written after a contribution by Alexander Belchenko on |
|
283 | 283 | the IPython user list. |
|
284 | 284 | |
|
285 | 285 | |
|
286 | 286 | Effective logging |
|
287 | 287 | ================= |
|
288 | 288 | |
|
289 | 289 | A very useful suggestion sent in by Robert Kern follows: |
|
290 | 290 | |
|
291 | 291 | I recently happened on a nifty way to keep tidy per-project log files. I |
|
292 | 292 | made a profile for my project (which is called "parkfield"):: |
|
293 | 293 | |
|
294 | 294 | include ipythonrc |
|
295 | 295 | |
|
296 | 296 | # cancel earlier logfile invocation: |
|
297 | 297 | |
|
298 | 298 | logfile '' |
|
299 | 299 | |
|
300 | 300 | execute import time |
|
301 | 301 | |
|
302 | 302 | execute __cmd = '/Users/kern/research/logfiles/parkfield-%s.log rotate' |
|
303 | 303 | |
|
304 | 304 | execute __IP.magic_logstart(__cmd % time.strftime('%Y-%m-%d')) |
|
305 | 305 | |
|
306 | 306 | I also added a shell alias for convenience:: |
|
307 | 307 | |
|
308 | 308 | alias parkfield="ipython -pylab -profile parkfield" |
|
309 | 309 | |
|
310 | 310 | Now I have a nice little directory with everything I ever type in, |
|
311 | 311 | organized by project and date. |
|
312 | 312 | |
|
313 | 313 | Contribute your own: If you have your own favorite tip on using IPython |
|
314 | 314 | efficiently for a certain task (especially things which can't be done in |
|
315 | 315 | the normal Python interpreter), don't hesitate to send it! |
|
316 | ||
|
317 |
@@ -1,232 +1,234 b'' | |||
|
1 | 1 | .. _overview: |
|
2 | 2 | |
|
3 | 3 | ============ |
|
4 | 4 | Introduction |
|
5 | 5 | ============ |
|
6 | 6 | |
|
7 | 7 | Overview |
|
8 | 8 | ======== |
|
9 | 9 | |
|
10 | 10 | One of Python's most useful features is its interactive interpreter. |
|
11 | 11 | This system allows very fast testing of ideas without the overhead of |
|
12 | 12 | creating test files as is typical in most programming languages. |
|
13 | 13 | However, the interpreter supplied with the standard Python distribution |
|
14 | 14 | is somewhat limited for extended interactive use. |
|
15 | 15 | |
|
16 | 16 | The goal of IPython is to create a comprehensive environment for |
|
17 | 17 | interactive and exploratory computing. To support this goal, IPython |
|
18 | 18 | has two main components: |
|
19 | 19 | |
|
20 | 20 | * An enhanced interactive Python shell. |
|
21 | 21 | * An architecture for interactive parallel computing. |
|
22 | 22 | |
|
23 | 23 | All of IPython is open source (released under the revised BSD license). |
|
24 | 24 | |
|
25 | 25 | Enhanced interactive Python shell |
|
26 | 26 | ================================= |
|
27 | 27 | |
|
28 | 28 | IPython's interactive shell (:command:`ipython`), has the following goals, |
|
29 | 29 | amongst others: |
|
30 | 30 | |
|
31 | 31 | 1. Provide an interactive shell superior to Python's default. IPython |
|
32 | 32 | has many features for object introspection, system shell access, |
|
33 | 33 | and its own special command system for adding functionality when |
|
34 | 34 | working interactively. It tries to be a very efficient environment |
|
35 | 35 | both for Python code development and for exploration of problems |
|
36 | 36 | using Python objects (in situations like data analysis). |
|
37 | 37 | |
|
38 | 38 | 2. Serve as an embeddable, ready to use interpreter for your own |
|
39 | 39 | programs. IPython can be started with a single call from inside |
|
40 | 40 | another program, providing access to the current namespace. This |
|
41 | 41 | can be very useful both for debugging purposes and for situations |
|
42 | 42 | where a blend of batch-processing and interactive exploration are |
|
43 | 43 | needed. New in the 0.9 version of IPython is a reusable wxPython |
|
44 | 44 | based IPython widget. |
|
45 | 45 | |
|
46 | 46 | 3. Offer a flexible framework which can be used as the base |
|
47 | 47 | environment for other systems with Python as the underlying |
|
48 | 48 | language. Specifically scientific environments like Mathematica, |
|
49 | 49 | IDL and Matlab inspired its design, but similar ideas can be |
|
50 | 50 | useful in many fields. |
|
51 | 51 | |
|
52 | 52 | 4. Allow interactive testing of threaded graphical toolkits. IPython |
|
53 | 53 | has support for interactive, non-blocking control of GTK, Qt and |
|
54 | 54 | WX applications via special threading flags. The normal Python |
|
55 | 55 | shell can only do this for Tkinter applications. |
|
56 | 56 | |
|
57 | 57 | Main features of the interactive shell |
|
58 | 58 | -------------------------------------- |
|
59 | 59 | |
|
60 | 60 | * Dynamic object introspection. One can access docstrings, function |
|
61 | 61 | definition prototypes, source code, source files and other details |
|
62 | 62 | of any object accessible to the interpreter with a single |
|
63 | 63 | keystroke (:samp:`?`, and using :samp:`??` provides additional detail). |
|
64 | 64 | |
|
65 | 65 | * Searching through modules and namespaces with :samp:`*` wildcards, both |
|
66 | 66 | when using the :samp:`?` system and via the :samp:`%psearch` command. |
|
67 | 67 | |
|
68 | 68 | * Completion in the local namespace, by typing :kbd:`TAB` at the prompt. |
|
69 | 69 | This works for keywords, modules, methods, variables and files in the |
|
70 | 70 | current directory. This is supported via the readline library, and |
|
71 | 71 | full access to configuring readline's behavior is provided. |
|
72 | 72 | Custom completers can be implemented easily for different purposes |
|
73 | 73 | (system commands, magic arguments etc.) |
|
74 | 74 | |
|
75 | 75 | * Numbered input/output prompts with command history (persistent |
|
76 | 76 | across sessions and tied to each profile), full searching in this |
|
77 | 77 | history and caching of all input and output. |
|
78 | 78 | |
|
79 | 79 | * User-extensible 'magic' commands. A set of commands prefixed with |
|
80 | 80 | :samp:`%` is available for controlling IPython itself and provides |
|
81 | 81 | directory control, namespace information and many aliases to |
|
82 | 82 | common system shell commands. |
|
83 | 83 | |
|
84 | 84 | * Alias facility for defining your own system aliases. |
|
85 | 85 | |
|
86 | 86 | * Complete system shell access. Lines starting with :samp:`!` are passed |
|
87 | 87 | directly to the system shell, and using :samp:`!!` or :samp:`var = !cmd` |
|
88 | 88 | captures shell output into python variables for further use. |
|
89 | 89 | |
|
90 | 90 | * Background execution of Python commands in a separate thread. |
|
91 | 91 | IPython has an internal job manager called jobs, and a |
|
92 | 92 | convenience backgrounding magic function called :samp:`%bg`. |
|
93 | 93 | |
|
94 | 94 | * The ability to expand python variables when calling the system shell. In a |
|
95 | 95 | shell command, any python variable prefixed with :samp:`$` is expanded. A |
|
96 | 96 | double :samp:`$$` allows passing a literal :samp:`$` to the shell (for access |
|
97 | 97 | to shell and environment variables like :envvar:`PATH`). |
|
98 | 98 | |
|
99 | 99 | * Filesystem navigation, via a magic :samp:`%cd` command, along with a |
|
100 | 100 | persistent bookmark system (using :samp:`%bookmark`) for fast access to |
|
101 | 101 | frequently visited directories. |
|
102 | 102 | |
|
103 | 103 | * A lightweight persistence framework via the :samp:`%store` command, which |
|
104 | 104 | allows you to save arbitrary Python variables. These get restored |
|
105 | 105 | automatically when your session restarts. |
|
106 | 106 | |
|
107 | 107 | * Automatic indentation (optional) of code as you type (through the |
|
108 | 108 | readline library). |
|
109 | 109 | |
|
110 | 110 | * Macro system for quickly re-executing multiple lines of previous |
|
111 | 111 | input with a single name. Macros can be stored persistently via |
|
112 | 112 | :samp:`%store` and edited via :samp:`%edit`. |
|
113 | 113 | |
|
114 | 114 | * Session logging (you can then later use these logs as code in your |
|
115 | 115 | programs). Logs can optionally timestamp all input, and also store |
|
116 | 116 | session output (marked as comments, so the log remains valid |
|
117 | 117 | Python source code). |
|
118 | 118 | |
|
119 | 119 | * Session restoring: logs can be replayed to restore a previous |
|
120 | 120 | session to the state where you left it. |
|
121 | 121 | |
|
122 | 122 | * Verbose and colored exception traceback printouts. Easier to parse |
|
123 | 123 | visually, and in verbose mode they produce a lot of useful |
|
124 | 124 | debugging information (basically a terminal version of the cgitb |
|
125 | 125 | module). |
|
126 | 126 | |
|
127 | 127 | * Auto-parentheses: callable objects can be executed without |
|
128 | 128 | parentheses: :samp:`sin 3` is automatically converted to :samp:`sin(3)`. |
|
129 | 129 | |
|
130 | 130 | * Auto-quoting: using :samp:`,`, or :samp:`;` as the first character forces |
|
131 | 131 | auto-quoting of the rest of the line: :samp:`,my_function a b` becomes |
|
132 | 132 | automatically :samp:`my_function("a","b")`, while :samp:`;my_function a b` |
|
133 | 133 | becomes :samp:`my_function("a b")`. |
|
134 | 134 | |
|
135 | 135 | * Extensible input syntax. You can define filters that pre-process |
|
136 | 136 | user input to simplify input in special situations. This allows |
|
137 | 137 | for example pasting multi-line code fragments which start with |
|
138 | 138 | :samp:`>>>` or :samp:`...` such as those from other python sessions or the |
|
139 | 139 | standard Python documentation. |
|
140 | 140 | |
|
141 | 141 | * Flexible configuration system. It uses a configuration file which |
|
142 | 142 | allows permanent setting of all command-line options, module |
|
143 | 143 | loading, code and file execution. The system allows recursive file |
|
144 | 144 | inclusion, so you can have a base file with defaults and layers |
|
145 | 145 | which load other customizations for particular projects. |
|
146 | 146 | |
|
147 | 147 | * Embeddable. You can call IPython as a python shell inside your own |
|
148 | 148 | python programs. This can be used both for debugging code or for |
|
149 | 149 | providing interactive abilities to your programs with knowledge |
|
150 | 150 | about the local namespaces (very useful in debugging and data |
|
151 | 151 | analysis situations). |
|
152 | 152 | |
|
153 | 153 | * Easy debugger access. You can set IPython to call up an enhanced version of |
|
154 | 154 | the Python debugger (pdb) every time there is an uncaught exception. This |
|
155 | 155 | drops you inside the code which triggered the exception with all the data |
|
156 | 156 | live and it is possible to navigate the stack to rapidly isolate the source |
|
157 | 157 | of a bug. The :samp:`%run` magic command (with the :samp:`-d` option) can run |
|
158 | 158 | any script under pdb's control, automatically setting initial breakpoints for |
|
159 | 159 | you. This version of pdb has IPython-specific improvements, including |
|
160 | 160 | tab-completion and traceback coloring support. For even easier debugger |
|
161 | 161 | access, try :samp:`%debug` after seeing an exception. winpdb is also |
|
162 | 162 | supported, see ipy_winpdb extension. |
|
163 | 163 | |
|
164 | 164 | * Profiler support. You can run single statements (similar to |
|
165 | 165 | :samp:`profile.run()`) or complete programs under the profiler's control. |
|
166 | 166 | While this is possible with standard cProfile or profile modules, |
|
167 | 167 | IPython wraps this functionality with magic commands (see :samp:`%prun` |
|
168 | 168 | and :samp:`%run -p`) convenient for rapid interactive work. |
|
169 | 169 | |
|
170 | 170 | * Doctest support. The special :samp:`%doctest_mode` command toggles a mode |
|
171 | 171 | that allows you to paste existing doctests (with leading :samp:`>>>` |
|
172 | 172 | prompts and whitespace) and uses doctest-compatible prompts and |
|
173 | 173 | output, so you can use IPython sessions as doctest code. |
|
174 | 174 | |
|
175 | 175 | Interactive parallel computing |
|
176 | 176 | ============================== |
|
177 | 177 | |
|
178 | 178 | Increasingly, parallel computer hardware, such as multicore CPUs, clusters and |
|
179 | 179 | supercomputers, is becoming ubiquitous. Over the last 3 years, we have |
|
180 | 180 | developed an architecture within IPython that allows such hardware to be used |
|
181 | 181 | quickly and easily from Python. Moreover, this architecture is designed to |
|
182 | 182 | support interactive and collaborative parallel computing. |
|
183 | 183 | |
|
184 | 184 | The main features of this system are: |
|
185 | 185 | |
|
186 | 186 | * Quickly parallelize Python code from an interactive Python/IPython session. |
|
187 | 187 | |
|
188 | 188 | * A flexible and dynamic process model that be deployed on anything from |
|
189 | 189 | multicore workstations to supercomputers. |
|
190 | 190 | |
|
191 | 191 | * An architecture that supports many different styles of parallelism, from |
|
192 | 192 | message passing to task farming. And all of these styles can be handled |
|
193 | 193 | interactively. |
|
194 | 194 | |
|
195 | 195 | * Both blocking and fully asynchronous interfaces. |
|
196 | 196 | |
|
197 | 197 | * High level APIs that enable many things to be parallelized in a few lines |
|
198 | 198 | of code. |
|
199 | 199 | |
|
200 | 200 | * Write parallel code that will run unchanged on everything from multicore |
|
201 | 201 | workstations to supercomputers. |
|
202 | 202 | |
|
203 | 203 | * Full integration with Message Passing libraries (MPI). |
|
204 | 204 | |
|
205 | 205 | * Capabilities based security model with full encryption of network connections. |
|
206 | 206 | |
|
207 | 207 | * Share live parallel jobs with other users securely. We call this |
|
208 | 208 | collaborative parallel computing. |
|
209 | 209 | |
|
210 | 210 | * Dynamically load balanced task farming system. |
|
211 | 211 | |
|
212 | 212 | * Robust error handling. Python exceptions raised in parallel execution are |
|
213 | 213 | gathered and presented to the top-level code. |
|
214 | 214 | |
|
215 | 215 | For more information, see our :ref:`overview <parallel_index>` of using IPython |
|
216 | 216 | for parallel computing. |
|
217 | 217 | |
|
218 | 218 | Portability and Python requirements |
|
219 | 219 | ----------------------------------- |
|
220 | 220 | |
|
221 |
As of the 0. |
|
|
222 | not begun to test IPython on Python 2.6 or 3.0, but we expect it will | |
|
223 | work with some minor changes. | |
|
221 | As of the 0.11 release, IPython works with either Python 2.5 or 2.6. | |
|
222 | Versions 0.9 and 0.10 worked with Python 2.4 as well. We have not begun | |
|
223 | the test and port IPython to 3.0. Our plan is to gradually drop Python 2.5 | |
|
224 | support and then begin the transition to strict 2.6 and 3.0. | |
|
224 | 225 | |
|
225 | 226 | IPython is known to work on the following operating systems: |
|
226 | 227 | |
|
227 | 228 | * Linux |
|
228 | 229 | * Most other Unix-like OSs (AIX, Solaris, BSD, etc.) |
|
229 | 230 | * Mac OS X |
|
230 | 231 | * Windows (CygWin, XP, Vista, etc.) |
|
231 | 232 | |
|
232 | See :ref:`here <install_index>` for instructions on how to install IPython. No newline at end of file | |
|
233 | See :ref:`here <install_index>` for instructions on how to install IPython. | |
|
234 |
@@ -1,15 +1,17 b'' | |||
|
1 | 1 | .. _parallel_index: |
|
2 | 2 | |
|
3 | 3 | ==================================== |
|
4 | 4 | Using IPython for parallel computing |
|
5 | 5 | ==================================== |
|
6 | 6 | |
|
7 | 7 | .. toctree:: |
|
8 | 8 | :maxdepth: 2 |
|
9 | 9 | |
|
10 | 10 | parallel_intro.txt |
|
11 | 11 | parallel_process.txt |
|
12 | 12 | parallel_multiengine.txt |
|
13 | 13 | parallel_task.txt |
|
14 | 14 | parallel_mpi.txt |
|
15 | 15 | parallel_security.txt |
|
16 | ||
|
17 |
@@ -1,176 +1,178 b'' | |||
|
1 | 1 | .. _parallelmpi: |
|
2 | 2 | |
|
3 | 3 | ======================= |
|
4 | 4 | Using MPI with IPython |
|
5 | 5 | ======================= |
|
6 | 6 | |
|
7 | 7 | Often, a parallel algorithm will require moving data between the engines. One |
|
8 | 8 | way of accomplishing this is by doing a pull and then a push using the |
|
9 | 9 | multiengine client. However, this will be slow as all the data has to go |
|
10 | 10 | through the controller to the client and then back through the controller, to |
|
11 | 11 | its final destination. |
|
12 | 12 | |
|
13 | 13 | A much better way of moving data between engines is to use a message passing |
|
14 | 14 | library, such as the Message Passing Interface (MPI) [MPI]_. IPython's |
|
15 | 15 | parallel computing architecture has been designed from the ground up to |
|
16 | 16 | integrate with MPI. This document describes how to use MPI with IPython. |
|
17 | 17 | |
|
18 | 18 | Additional installation requirements |
|
19 | 19 | ==================================== |
|
20 | 20 | |
|
21 | 21 | If you want to use MPI with IPython, you will need to install: |
|
22 | 22 | |
|
23 | 23 | * A standard MPI implementation such as OpenMPI [OpenMPI]_ or MPICH. |
|
24 | 24 | * The mpi4py [mpi4py]_ package. |
|
25 | 25 | |
|
26 | 26 | .. note:: |
|
27 | 27 | |
|
28 | 28 | The mpi4py package is not a strict requirement. However, you need to |
|
29 | 29 | have *some* way of calling MPI from Python. You also need some way of |
|
30 | 30 | making sure that :func:`MPI_Init` is called when the IPython engines start |
|
31 | 31 | up. There are a number of ways of doing this and a good number of |
|
32 | 32 | associated subtleties. We highly recommend just using mpi4py as it |
|
33 | 33 | takes care of most of these problems. If you want to do something |
|
34 | 34 | different, let us know and we can help you get started. |
|
35 | 35 | |
|
36 | 36 | Starting the engines with MPI enabled |
|
37 | 37 | ===================================== |
|
38 | 38 | |
|
39 | 39 | To use code that calls MPI, there are typically two things that MPI requires. |
|
40 | 40 | |
|
41 | 41 | 1. The process that wants to call MPI must be started using |
|
42 | 42 | :command:`mpiexec` or a batch system (like PBS) that has MPI support. |
|
43 | 43 | 2. Once the process starts, it must call :func:`MPI_Init`. |
|
44 | 44 | |
|
45 | 45 | There are a couple of ways that you can start the IPython engines and get |
|
46 | 46 | these things to happen. |
|
47 | 47 | |
|
48 | 48 | Automatic starting using :command:`mpiexec` and :command:`ipcluster` |
|
49 | 49 | -------------------------------------------------------------------- |
|
50 | 50 | |
|
51 | 51 | The easiest approach is to use the `mpiexec` mode of :command:`ipcluster`, |
|
52 | 52 | which will first start a controller and then a set of engines using |
|
53 | 53 | :command:`mpiexec`:: |
|
54 | 54 | |
|
55 | 55 | $ ipcluster mpiexec -n 4 |
|
56 | 56 | |
|
57 | 57 | This approach is best as interrupting :command:`ipcluster` will automatically |
|
58 | 58 | stop and clean up the controller and engines. |
|
59 | 59 | |
|
60 | 60 | Manual starting using :command:`mpiexec` |
|
61 | 61 | ---------------------------------------- |
|
62 | 62 | |
|
63 | 63 | If you want to start the IPython engines using the :command:`mpiexec`, just |
|
64 | 64 | do:: |
|
65 | 65 | |
|
66 | 66 | $ mpiexec -n 4 ipengine --mpi=mpi4py |
|
67 | 67 | |
|
68 | 68 | This requires that you already have a controller running and that the FURL |
|
69 | 69 | files for the engines are in place. We also have built in support for |
|
70 | 70 | PyTrilinos [PyTrilinos]_, which can be used (assuming is installed) by |
|
71 | 71 | starting the engines with:: |
|
72 | 72 | |
|
73 | 73 | mpiexec -n 4 ipengine --mpi=pytrilinos |
|
74 | 74 | |
|
75 | 75 | Automatic starting using PBS and :command:`ipcluster` |
|
76 | 76 | ----------------------------------------------------- |
|
77 | 77 | |
|
78 | 78 | The :command:`ipcluster` command also has built-in integration with PBS. For |
|
79 | 79 | more information on this approach, see our documentation on :ref:`ipcluster |
|
80 | 80 | <parallel_process>`. |
|
81 | 81 | |
|
82 | 82 | Actually using MPI |
|
83 | 83 | ================== |
|
84 | 84 | |
|
85 | 85 | Once the engines are running with MPI enabled, you are ready to go. You can |
|
86 | 86 | now call any code that uses MPI in the IPython engines. And, all of this can |
|
87 | 87 | be done interactively. Here we show a simple example that uses mpi4py |
|
88 | 88 | [mpi4py]_. |
|
89 | 89 | |
|
90 | 90 | First, lets define a simply function that uses MPI to calculate the sum of a |
|
91 | 91 | distributed array. Save the following text in a file called :file:`psum.py`: |
|
92 | 92 | |
|
93 | 93 | .. sourcecode:: python |
|
94 | 94 | |
|
95 | 95 | from mpi4py import MPI |
|
96 | 96 | import numpy as np |
|
97 | 97 | |
|
98 | 98 | def psum(a): |
|
99 | 99 | s = np.sum(a) |
|
100 | 100 | return MPI.COMM_WORLD.Allreduce(s,MPI.SUM) |
|
101 | 101 | |
|
102 | 102 | Now, start an IPython cluster in the same directory as :file:`psum.py`:: |
|
103 | 103 | |
|
104 | 104 | $ ipcluster mpiexec -n 4 |
|
105 | 105 | |
|
106 | 106 | Finally, connect to the cluster and use this function interactively. In this |
|
107 | 107 | case, we create a random array on each engine and sum up all the random arrays |
|
108 | 108 | using our :func:`psum` function: |
|
109 | 109 | |
|
110 | 110 | .. sourcecode:: ipython |
|
111 | 111 | |
|
112 | 112 | In [1]: from IPython.kernel import client |
|
113 | 113 | |
|
114 | 114 | In [2]: mec = client.MultiEngineClient() |
|
115 | 115 | |
|
116 | 116 | In [3]: mec.activate() |
|
117 | 117 | |
|
118 | 118 | In [4]: px import numpy as np |
|
119 | 119 | Parallel execution on engines: all |
|
120 | 120 | Out[4]: |
|
121 | 121 | <Results List> |
|
122 | 122 | [0] In [13]: import numpy as np |
|
123 | 123 | [1] In [13]: import numpy as np |
|
124 | 124 | [2] In [13]: import numpy as np |
|
125 | 125 | [3] In [13]: import numpy as np |
|
126 | 126 | |
|
127 | 127 | In [6]: px a = np.random.rand(100) |
|
128 | 128 | Parallel execution on engines: all |
|
129 | 129 | Out[6]: |
|
130 | 130 | <Results List> |
|
131 | 131 | [0] In [15]: a = np.random.rand(100) |
|
132 | 132 | [1] In [15]: a = np.random.rand(100) |
|
133 | 133 | [2] In [15]: a = np.random.rand(100) |
|
134 | 134 | [3] In [15]: a = np.random.rand(100) |
|
135 | 135 | |
|
136 | 136 | In [7]: px from psum import psum |
|
137 | 137 | Parallel execution on engines: all |
|
138 | 138 | Out[7]: |
|
139 | 139 | <Results List> |
|
140 | 140 | [0] In [16]: from psum import psum |
|
141 | 141 | [1] In [16]: from psum import psum |
|
142 | 142 | [2] In [16]: from psum import psum |
|
143 | 143 | [3] In [16]: from psum import psum |
|
144 | 144 | |
|
145 | 145 | In [8]: px s = psum(a) |
|
146 | 146 | Parallel execution on engines: all |
|
147 | 147 | Out[8]: |
|
148 | 148 | <Results List> |
|
149 | 149 | [0] In [17]: s = psum(a) |
|
150 | 150 | [1] In [17]: s = psum(a) |
|
151 | 151 | [2] In [17]: s = psum(a) |
|
152 | 152 | [3] In [17]: s = psum(a) |
|
153 | 153 | |
|
154 | 154 | In [9]: px print s |
|
155 | 155 | Parallel execution on engines: all |
|
156 | 156 | Out[9]: |
|
157 | 157 | <Results List> |
|
158 | 158 | [0] In [18]: print s |
|
159 | 159 | [0] Out[18]: 187.451545803 |
|
160 | 160 | |
|
161 | 161 | [1] In [18]: print s |
|
162 | 162 | [1] Out[18]: 187.451545803 |
|
163 | 163 | |
|
164 | 164 | [2] In [18]: print s |
|
165 | 165 | [2] Out[18]: 187.451545803 |
|
166 | 166 | |
|
167 | 167 | [3] In [18]: print s |
|
168 | 168 | [3] Out[18]: 187.451545803 |
|
169 | 169 | |
|
170 | 170 | Any Python code that makes calls to MPI can be used in this manner, including |
|
171 | 171 | compiled C, C++ and Fortran libraries that have been exposed to Python. |
|
172 | 172 | |
|
173 | 173 | .. [MPI] Message Passing Interface. http://www-unix.mcs.anl.gov/mpi/ |
|
174 | 174 | .. [mpi4py] MPI for Python. mpi4py: http://mpi4py.scipy.org/ |
|
175 | 175 | .. [OpenMPI] Open MPI. http://www.open-mpi.org/ |
|
176 | 176 | .. [PyTrilinos] PyTrilinos. http://trilinos.sandia.gov/packages/pytrilinos/ |
|
177 | ||
|
178 |
@@ -1,387 +1,389 b'' | |||
|
1 | 1 | .. _parallel_process: |
|
2 | 2 | |
|
3 | 3 | =========================================== |
|
4 | 4 | Starting the IPython controller and engines |
|
5 | 5 | =========================================== |
|
6 | 6 | |
|
7 | 7 | To use IPython for parallel computing, you need to start one instance of |
|
8 | 8 | the controller and one or more instances of the engine. The controller |
|
9 | 9 | and each engine can run on different machines or on the same machine. |
|
10 | 10 | Because of this, there are many different possibilities. |
|
11 | 11 | |
|
12 | 12 | Broadly speaking, there are two ways of going about starting a controller and engines: |
|
13 | 13 | |
|
14 | 14 | * In an automated manner using the :command:`ipcluster` command. |
|
15 | 15 | * In a more manual way using the :command:`ipcontroller` and |
|
16 | 16 | :command:`ipengine` commands. |
|
17 | 17 | |
|
18 | 18 | This document describes both of these methods. We recommend that new users |
|
19 | 19 | start with the :command:`ipcluster` command as it simplifies many common usage |
|
20 | 20 | cases. |
|
21 | 21 | |
|
22 | 22 | General considerations |
|
23 | 23 | ====================== |
|
24 | 24 | |
|
25 | 25 | Before delving into the details about how you can start a controller and |
|
26 | 26 | engines using the various methods, we outline some of the general issues that |
|
27 | 27 | come up when starting the controller and engines. These things come up no |
|
28 | 28 | matter which method you use to start your IPython cluster. |
|
29 | 29 | |
|
30 | 30 | Let's say that you want to start the controller on ``host0`` and engines on |
|
31 | 31 | hosts ``host1``-``hostn``. The following steps are then required: |
|
32 | 32 | |
|
33 | 33 | 1. Start the controller on ``host0`` by running :command:`ipcontroller` on |
|
34 | 34 | ``host0``. |
|
35 | 35 | 2. Move the FURL file (:file:`ipcontroller-engine.furl`) created by the |
|
36 | 36 | controller from ``host0`` to hosts ``host1``-``hostn``. |
|
37 | 37 | 3. Start the engines on hosts ``host1``-``hostn`` by running |
|
38 | 38 | :command:`ipengine`. This command has to be told where the FURL file |
|
39 | 39 | (:file:`ipcontroller-engine.furl`) is located. |
|
40 | 40 | |
|
41 | 41 | At this point, the controller and engines will be connected. By default, the |
|
42 | 42 | FURL files created by the controller are put into the |
|
43 | 43 | :file:`~/.ipython/security` directory. If the engines share a filesystem with |
|
44 | 44 | the controller, step 2 can be skipped as the engines will automatically look |
|
45 | 45 | at that location. |
|
46 | 46 | |
|
47 | 47 | The final step required required to actually use the running controller from a |
|
48 | 48 | client is to move the FURL files :file:`ipcontroller-mec.furl` and |
|
49 | 49 | :file:`ipcontroller-tc.furl` from ``host0`` to the host where the clients will |
|
50 | 50 | be run. If these file are put into the :file:`~/.ipython/security` directory |
|
51 | 51 | of the client's host, they will be found automatically. Otherwise, the full |
|
52 | 52 | path to them has to be passed to the client's constructor. |
|
53 | 53 | |
|
54 | 54 | Using :command:`ipcluster` |
|
55 | 55 | ========================== |
|
56 | 56 | |
|
57 | 57 | The :command:`ipcluster` command provides a simple way of starting a |
|
58 | 58 | controller and engines in the following situations: |
|
59 | 59 | |
|
60 | 60 | 1. When the controller and engines are all run on localhost. This is useful |
|
61 | 61 | for testing or running on a multicore computer. |
|
62 | 62 | 2. When engines are started using the :command:`mpirun` command that comes |
|
63 | 63 | with most MPI [MPI]_ implementations |
|
64 | 64 | 3. When engines are started using the PBS [PBS]_ batch system. |
|
65 | 65 | 4. When the controller is started on localhost and the engines are started on |
|
66 | 66 | remote nodes using :command:`ssh`. |
|
67 | 67 | |
|
68 | 68 | .. note:: |
|
69 | 69 | |
|
70 | 70 | It is also possible for advanced users to add support to |
|
71 | 71 | :command:`ipcluster` for starting controllers and engines using other |
|
72 | 72 | methods (like Sun's Grid Engine for example). |
|
73 | 73 | |
|
74 | 74 | .. note:: |
|
75 | 75 | |
|
76 | 76 | Currently :command:`ipcluster` requires that the |
|
77 | 77 | :file:`~/.ipython/security` directory live on a shared filesystem that is |
|
78 | 78 | seen by both the controller and engines. If you don't have a shared file |
|
79 | 79 | system you will need to use :command:`ipcontroller` and |
|
80 | 80 | :command:`ipengine` directly. This constraint can be relaxed if you are |
|
81 | 81 | using the :command:`ssh` method to start the cluster. |
|
82 | 82 | |
|
83 | 83 | Underneath the hood, :command:`ipcluster` just uses :command:`ipcontroller` |
|
84 | 84 | and :command:`ipengine` to perform the steps described above. |
|
85 | 85 | |
|
86 | 86 | Using :command:`ipcluster` in local mode |
|
87 | 87 | ---------------------------------------- |
|
88 | 88 | |
|
89 | 89 | To start one controller and 4 engines on localhost, just do:: |
|
90 | 90 | |
|
91 | 91 | $ ipcluster local -n 4 |
|
92 | 92 | |
|
93 | 93 | To see other command line options for the local mode, do:: |
|
94 | 94 | |
|
95 | 95 | $ ipcluster local -h |
|
96 | 96 | |
|
97 | 97 | Using :command:`ipcluster` in mpiexec/mpirun mode |
|
98 | 98 | ------------------------------------------------- |
|
99 | 99 | |
|
100 | 100 | The mpiexec/mpirun mode is useful if you: |
|
101 | 101 | |
|
102 | 102 | 1. Have MPI installed. |
|
103 | 103 | 2. Your systems are configured to use the :command:`mpiexec` or |
|
104 | 104 | :command:`mpirun` commands to start MPI processes. |
|
105 | 105 | |
|
106 | 106 | .. note:: |
|
107 | 107 | |
|
108 | 108 | The preferred command to use is :command:`mpiexec`. However, we also |
|
109 | 109 | support :command:`mpirun` for backwards compatibility. The underlying |
|
110 | 110 | logic used is exactly the same, the only difference being the name of the |
|
111 | 111 | command line program that is called. |
|
112 | 112 | |
|
113 | 113 | If these are satisfied, you can start an IPython cluster using:: |
|
114 | 114 | |
|
115 | 115 | $ ipcluster mpiexec -n 4 |
|
116 | 116 | |
|
117 | 117 | This does the following: |
|
118 | 118 | |
|
119 | 119 | 1. Starts the IPython controller on current host. |
|
120 | 120 | 2. Uses :command:`mpiexec` to start 4 engines. |
|
121 | 121 | |
|
122 | 122 | On newer MPI implementations (such as OpenMPI), this will work even if you |
|
123 | 123 | don't make any calls to MPI or call :func:`MPI_Init`. However, older MPI |
|
124 | 124 | implementations actually require each process to call :func:`MPI_Init` upon |
|
125 | 125 | starting. The easiest way of having this done is to install the mpi4py |
|
126 | 126 | [mpi4py]_ package and then call ipcluster with the ``--mpi`` option:: |
|
127 | 127 | |
|
128 | 128 | $ ipcluster mpiexec -n 4 --mpi=mpi4py |
|
129 | 129 | |
|
130 | 130 | Unfortunately, even this won't work for some MPI implementations. If you are |
|
131 | 131 | having problems with this, you will likely have to use a custom Python |
|
132 | 132 | executable that itself calls :func:`MPI_Init` at the appropriate time. |
|
133 | 133 | Fortunately, mpi4py comes with such a custom Python executable that is easy to |
|
134 | 134 | install and use. However, this custom Python executable approach will not work |
|
135 | 135 | with :command:`ipcluster` currently. |
|
136 | 136 | |
|
137 | 137 | Additional command line options for this mode can be found by doing:: |
|
138 | 138 | |
|
139 | 139 | $ ipcluster mpiexec -h |
|
140 | 140 | |
|
141 | 141 | More details on using MPI with IPython can be found :ref:`here <parallelmpi>`. |
|
142 | 142 | |
|
143 | 143 | |
|
144 | 144 | Using :command:`ipcluster` in PBS mode |
|
145 | 145 | -------------------------------------- |
|
146 | 146 | |
|
147 | 147 | The PBS mode uses the Portable Batch System [PBS]_ to start the engines. To |
|
148 | 148 | use this mode, you first need to create a PBS script template that will be |
|
149 | 149 | used to start the engines. Here is a sample PBS script template: |
|
150 | 150 | |
|
151 | 151 | .. sourcecode:: bash |
|
152 | 152 | |
|
153 | 153 | #PBS -N ipython |
|
154 | 154 | #PBS -j oe |
|
155 | 155 | #PBS -l walltime=00:10:00 |
|
156 | 156 | #PBS -l nodes=${n/4}:ppn=4 |
|
157 | 157 | #PBS -q parallel |
|
158 | 158 | |
|
159 | 159 | cd $$PBS_O_WORKDIR |
|
160 | 160 | export PATH=$$HOME/usr/local/bin |
|
161 | 161 | export PYTHONPATH=$$HOME/usr/local/lib/python2.4/site-packages |
|
162 | 162 | /usr/local/bin/mpiexec -n ${n} ipengine --logfile=$$PBS_O_WORKDIR/ipengine |
|
163 | 163 | |
|
164 | 164 | There are a few important points about this template: |
|
165 | 165 | |
|
166 | 166 | 1. This template will be rendered at runtime using IPython's :mod:`Itpl` |
|
167 | 167 | template engine. |
|
168 | 168 | |
|
169 | 169 | 2. Instead of putting in the actual number of engines, use the notation |
|
170 | 170 | ``${n}`` to indicate the number of engines to be started. You can also uses |
|
171 | 171 | expressions like ``${n/4}`` in the template to indicate the number of |
|
172 | 172 | nodes. |
|
173 | 173 | |
|
174 | 174 | 3. Because ``$`` is a special character used by the template engine, you must |
|
175 | 175 | escape any ``$`` by using ``$$``. This is important when referring to |
|
176 | 176 | environment variables in the template. |
|
177 | 177 | |
|
178 | 178 | 4. Any options to :command:`ipengine` should be given in the batch script |
|
179 | 179 | template. |
|
180 | 180 | |
|
181 | 181 | 5. Depending on the configuration of you system, you may have to set |
|
182 | 182 | environment variables in the script template. |
|
183 | 183 | |
|
184 | 184 | Once you have created such a script, save it with a name like |
|
185 | 185 | :file:`pbs.template`. Now you are ready to start your job:: |
|
186 | 186 | |
|
187 | 187 | $ ipcluster pbs -n 128 --pbs-script=pbs.template |
|
188 | 188 | |
|
189 | 189 | Additional command line options for this mode can be found by doing:: |
|
190 | 190 | |
|
191 | 191 | $ ipcluster pbs -h |
|
192 | 192 | |
|
193 | 193 | Using :command:`ipcluster` in SSH mode |
|
194 | 194 | -------------------------------------- |
|
195 | 195 | |
|
196 | 196 | The SSH mode uses :command:`ssh` to execute :command:`ipengine` on remote |
|
197 | 197 | nodes and the :command:`ipcontroller` on localhost. |
|
198 | 198 | |
|
199 | 199 | When using using this mode it highly recommended that you have set up SSH keys |
|
200 | 200 | and are using ssh-agent [SSH]_ for password-less logins. |
|
201 | 201 | |
|
202 | 202 | To use this mode you need a python file describing the cluster, here is an |
|
203 | 203 | example of such a "clusterfile": |
|
204 | 204 | |
|
205 | 205 | .. sourcecode:: python |
|
206 | 206 | |
|
207 | 207 | send_furl = True |
|
208 | 208 | engines = { 'host1.example.com' : 2, |
|
209 | 209 | 'host2.example.com' : 5, |
|
210 | 210 | 'host3.example.com' : 1, |
|
211 | 211 | 'host4.example.com' : 8 } |
|
212 | 212 | |
|
213 | 213 | Since this is a regular python file usual python syntax applies. Things to |
|
214 | 214 | note: |
|
215 | 215 | |
|
216 | 216 | * The `engines` dict, where the keys is the host we want to run engines on and |
|
217 | 217 | the value is the number of engines to run on that host. |
|
218 | 218 | * send_furl can either be `True` or `False`, if `True` it will copy over the |
|
219 | 219 | furl needed for :command:`ipengine` to each host. |
|
220 | 220 | |
|
221 | 221 | The ``--clusterfile`` command line option lets you specify the file to use for |
|
222 | 222 | the cluster definition. Once you have your cluster file and you can |
|
223 | 223 | :command:`ssh` into the remote hosts with out an password you are ready to |
|
224 | 224 | start your cluster like so: |
|
225 | 225 | |
|
226 | 226 | .. sourcecode:: bash |
|
227 | 227 | |
|
228 | 228 | $ ipcluster ssh --clusterfile /path/to/my/clusterfile.py |
|
229 | 229 | |
|
230 | 230 | |
|
231 | 231 | Two helper shell scripts are used to start and stop :command:`ipengine` on |
|
232 | 232 | remote hosts: |
|
233 | 233 | |
|
234 | 234 | * sshx.sh |
|
235 | 235 | * engine_killer.sh |
|
236 | 236 | |
|
237 | 237 | Defaults for both of these are contained in the source code for |
|
238 | 238 | :command:`ipcluster`. The default scripts are written to a local file in a |
|
239 | 239 | tmep directory and then copied to a temp directory on the remote host and |
|
240 | 240 | executed from there. On most Unix, Linux and OS X systems this is /tmp. |
|
241 | 241 | |
|
242 | 242 | The default sshx.sh is the following: |
|
243 | 243 | |
|
244 | 244 | .. sourcecode:: bash |
|
245 | 245 | |
|
246 | 246 | #!/bin/sh |
|
247 | 247 | "$@" &> /dev/null & |
|
248 | 248 | echo $! |
|
249 | 249 | |
|
250 | 250 | If you want to use a custom sshx.sh script you need to use the ``--sshx`` |
|
251 | 251 | option and specify the file to use. Using a custom sshx.sh file could be |
|
252 | 252 | helpful when you need to setup the environment on the remote host before |
|
253 | 253 | executing :command:`ipengine`. |
|
254 | 254 | |
|
255 | 255 | For a detailed options list: |
|
256 | 256 | |
|
257 | 257 | .. sourcecode:: bash |
|
258 | 258 | |
|
259 | 259 | $ ipcluster ssh -h |
|
260 | 260 | |
|
261 | 261 | Current limitations of the SSH mode of :command:`ipcluster` are: |
|
262 | 262 | |
|
263 | 263 | * Untested on Windows. Would require a working :command:`ssh` on Windows. |
|
264 | 264 | Also, we are using shell scripts to setup and execute commands on remote |
|
265 | 265 | hosts. |
|
266 | 266 | * :command:`ipcontroller` is started on localhost, with no option to start it |
|
267 | 267 | on a remote node. |
|
268 | 268 | |
|
269 | 269 | Using the :command:`ipcontroller` and :command:`ipengine` commands |
|
270 | 270 | ================================================================== |
|
271 | 271 | |
|
272 | 272 | It is also possible to use the :command:`ipcontroller` and :command:`ipengine` |
|
273 | 273 | commands to start your controller and engines. This approach gives you full |
|
274 | 274 | control over all aspects of the startup process. |
|
275 | 275 | |
|
276 | 276 | Starting the controller and engine on your local machine |
|
277 | 277 | -------------------------------------------------------- |
|
278 | 278 | |
|
279 | 279 | To use :command:`ipcontroller` and :command:`ipengine` to start things on your |
|
280 | 280 | local machine, do the following. |
|
281 | 281 | |
|
282 | 282 | First start the controller:: |
|
283 | 283 | |
|
284 | 284 | $ ipcontroller |
|
285 | 285 | |
|
286 | 286 | Next, start however many instances of the engine you want using (repeatedly) |
|
287 | 287 | the command:: |
|
288 | 288 | |
|
289 | 289 | $ ipengine |
|
290 | 290 | |
|
291 | 291 | The engines should start and automatically connect to the controller using the |
|
292 | 292 | FURL files in :file:`~./ipython/security`. You are now ready to use the |
|
293 | 293 | controller and engines from IPython. |
|
294 | 294 | |
|
295 | 295 | .. warning:: |
|
296 | 296 | |
|
297 | 297 | The order of the above operations is very important. You *must* |
|
298 | 298 | start the controller before the engines, since the engines connect |
|
299 | 299 | to the controller as they get started. |
|
300 | 300 | |
|
301 | 301 | .. note:: |
|
302 | 302 | |
|
303 | 303 | On some platforms (OS X), to put the controller and engine into the |
|
304 | 304 | background you may need to give these commands in the form ``(ipcontroller |
|
305 | 305 | &)`` and ``(ipengine &)`` (with the parentheses) for them to work |
|
306 | 306 | properly. |
|
307 | 307 | |
|
308 | 308 | Starting the controller and engines on different hosts |
|
309 | 309 | ------------------------------------------------------ |
|
310 | 310 | |
|
311 | 311 | When the controller and engines are running on different hosts, things are |
|
312 | 312 | slightly more complicated, but the underlying ideas are the same: |
|
313 | 313 | |
|
314 | 314 | 1. Start the controller on a host using :command:`ipcontroller`. |
|
315 | 315 | 2. Copy :file:`ipcontroller-engine.furl` from :file:`~./ipython/security` on |
|
316 | 316 | the controller's host to the host where the engines will run. |
|
317 | 317 | 3. Use :command:`ipengine` on the engine's hosts to start the engines. |
|
318 | 318 | |
|
319 | 319 | The only thing you have to be careful of is to tell :command:`ipengine` where |
|
320 | 320 | the :file:`ipcontroller-engine.furl` file is located. There are two ways you |
|
321 | 321 | can do this: |
|
322 | 322 | |
|
323 | 323 | * Put :file:`ipcontroller-engine.furl` in the :file:`~./ipython/security` |
|
324 | 324 | directory on the engine's host, where it will be found automatically. |
|
325 | 325 | * Call :command:`ipengine` with the ``--furl-file=full_path_to_the_file`` |
|
326 | 326 | flag. |
|
327 | 327 | |
|
328 | 328 | The ``--furl-file`` flag works like this:: |
|
329 | 329 | |
|
330 | 330 | $ ipengine --furl-file=/path/to/my/ipcontroller-engine.furl |
|
331 | 331 | |
|
332 | 332 | .. note:: |
|
333 | 333 | |
|
334 | 334 | If the controller's and engine's hosts all have a shared file system |
|
335 | 335 | (:file:`~./ipython/security` is the same on all of them), then things |
|
336 | 336 | will just work! |
|
337 | 337 | |
|
338 | 338 | Make FURL files persistent |
|
339 | 339 | --------------------------- |
|
340 | 340 | |
|
341 | 341 | At fist glance it may seem that that managing the FURL files is a bit |
|
342 | 342 | annoying. Going back to the house and key analogy, copying the FURL around |
|
343 | 343 | each time you start the controller is like having to make a new key every time |
|
344 | 344 | you want to unlock the door and enter your house. As with your house, you want |
|
345 | 345 | to be able to create the key (or FURL file) once, and then simply use it at |
|
346 | 346 | any point in the future. |
|
347 | 347 | |
|
348 | 348 | This is possible, but before you do this, you **must** remove any old FURL |
|
349 | 349 | files in the :file:`~/.ipython/security` directory. |
|
350 | 350 | |
|
351 | 351 | .. warning:: |
|
352 | 352 | |
|
353 | 353 | You **must** remove old FURL files before using persistent FURL files. |
|
354 | 354 | |
|
355 | 355 | Then, The only thing you have to do is decide what ports the controller will |
|
356 | 356 | listen on for the engines and clients. This is done as follows:: |
|
357 | 357 | |
|
358 | 358 | $ ipcontroller -r --client-port=10101 --engine-port=10102 |
|
359 | 359 | |
|
360 | 360 | These options also work with all of the various modes of |
|
361 | 361 | :command:`ipcluster`:: |
|
362 | 362 | |
|
363 | 363 | $ ipcluster local -n 2 -r --client-port=10101 --engine-port=10102 |
|
364 | 364 | |
|
365 | 365 | Then, just copy the furl files over the first time and you are set. You can |
|
366 | 366 | start and stop the controller and engines any many times as you want in the |
|
367 | 367 | future, just make sure to tell the controller to use the *same* ports. |
|
368 | 368 | |
|
369 | 369 | .. note:: |
|
370 | 370 | |
|
371 | 371 | You may ask the question: what ports does the controller listen on if you |
|
372 | 372 | don't tell is to use specific ones? The default is to use high random port |
|
373 | 373 | numbers. We do this for two reasons: i) to increase security through |
|
374 | 374 | obscurity and ii) to multiple controllers on a given host to start and |
|
375 | 375 | automatically use different ports. |
|
376 | 376 | |
|
377 | 377 | Log files |
|
378 | 378 | --------- |
|
379 | 379 | |
|
380 | 380 | All of the components of IPython have log files associated with them. |
|
381 | 381 | These log files can be extremely useful in debugging problems with |
|
382 | 382 | IPython and can be found in the directory :file:`~/.ipython/log`. Sending |
|
383 | 383 | the log files to us will often help us to debug any problems. |
|
384 | 384 | |
|
385 | 385 | |
|
386 | 386 | .. [PBS] Portable Batch System. http://www.openpbs.org/ |
|
387 | 387 | .. [SSH] SSH-Agent http://en.wikipedia.org/wiki/Ssh-agent |
|
388 | ||
|
389 |
@@ -1,364 +1,366 b'' | |||
|
1 | 1 | .. _parallelsecurity: |
|
2 | 2 | |
|
3 | 3 | =========================== |
|
4 | 4 | Security details of IPython |
|
5 | 5 | =========================== |
|
6 | 6 | |
|
7 | 7 | IPython's :mod:`IPython.kernel` package exposes the full power of the Python |
|
8 | 8 | interpreter over a TCP/IP network for the purposes of parallel computing. This |
|
9 | 9 | feature brings up the important question of IPython's security model. This |
|
10 | 10 | document gives details about this model and how it is implemented in IPython's |
|
11 | 11 | architecture. |
|
12 | 12 | |
|
13 | 13 | Processs and network topology |
|
14 | 14 | ============================= |
|
15 | 15 | |
|
16 | 16 | To enable parallel computing, IPython has a number of different processes that |
|
17 | 17 | run. These processes are discussed at length in the IPython documentation and |
|
18 | 18 | are summarized here: |
|
19 | 19 | |
|
20 | 20 | * The IPython *engine*. This process is a full blown Python |
|
21 | 21 | interpreter in which user code is executed. Multiple |
|
22 | 22 | engines are started to make parallel computing possible. |
|
23 | 23 | * The IPython *controller*. This process manages a set of |
|
24 | 24 | engines, maintaining a queue for each and presenting |
|
25 | 25 | an asynchronous interface to the set of engines. |
|
26 | 26 | * The IPython *client*. This process is typically an |
|
27 | 27 | interactive Python process that is used to coordinate the |
|
28 | 28 | engines to get a parallel computation done. |
|
29 | 29 | |
|
30 | 30 | Collectively, these three processes are called the IPython *kernel*. |
|
31 | 31 | |
|
32 | 32 | These three processes communicate over TCP/IP connections with a well defined |
|
33 | 33 | topology. The IPython controller is the only process that listens on TCP/IP |
|
34 | 34 | sockets. Upon starting, an engine connects to a controller and registers |
|
35 | 35 | itself with the controller. These engine/controller TCP/IP connections persist |
|
36 | 36 | for the lifetime of each engine. |
|
37 | 37 | |
|
38 | 38 | The IPython client also connects to the controller using one or more TCP/IP |
|
39 | 39 | connections. These connections persist for the lifetime of the client only. |
|
40 | 40 | |
|
41 | 41 | A given IPython controller and set of engines typically has a relatively short |
|
42 | 42 | lifetime. Typically this lifetime corresponds to the duration of a single |
|
43 | 43 | parallel simulation performed by a single user. Finally, the controller, |
|
44 | 44 | engines and client processes typically execute with the permissions of that |
|
45 | 45 | same user. More specifically, the controller and engines are *not* executed as |
|
46 | 46 | root or with any other superuser permissions. |
|
47 | 47 | |
|
48 | 48 | Application logic |
|
49 | 49 | ================= |
|
50 | 50 | |
|
51 | 51 | When running the IPython kernel to perform a parallel computation, a user |
|
52 | 52 | utilizes the IPython client to send Python commands and data through the |
|
53 | 53 | IPython controller to the IPython engines, where those commands are executed |
|
54 | 54 | and the data processed. The design of IPython ensures that the client is the |
|
55 | 55 | only access point for the capabilities of the engines. That is, the only way |
|
56 | 56 | of addressing the engines is through a client. |
|
57 | 57 | |
|
58 | 58 | A user can utilize the client to instruct the IPython engines to execute |
|
59 | 59 | arbitrary Python commands. These Python commands can include calls to the |
|
60 | 60 | system shell, access the filesystem, etc., as required by the user's |
|
61 | 61 | application code. From this perspective, when a user runs an IPython engine on |
|
62 | 62 | a host, that engine has the same capabilities and permissions as the user |
|
63 | 63 | themselves (as if they were logged onto the engine's host with a terminal). |
|
64 | 64 | |
|
65 | 65 | Secure network connections |
|
66 | 66 | ========================== |
|
67 | 67 | |
|
68 | 68 | Overview |
|
69 | 69 | -------- |
|
70 | 70 | |
|
71 | 71 | All TCP/IP connections between the client and controller as well as the |
|
72 | 72 | engines and controller are fully encrypted and authenticated. This section |
|
73 | 73 | describes the details of the encryption and authentication approached used |
|
74 | 74 | within IPython. |
|
75 | 75 | |
|
76 | 76 | IPython uses the Foolscap network protocol [Foolscap]_ for all communications |
|
77 | 77 | between processes. Thus, the details of IPython's security model are directly |
|
78 | 78 | related to those of Foolscap. Thus, much of the following discussion is |
|
79 | 79 | actually just a discussion of the security that is built in to Foolscap. |
|
80 | 80 | |
|
81 | 81 | Encryption |
|
82 | 82 | ---------- |
|
83 | 83 | |
|
84 | 84 | For encryption purposes, IPython and Foolscap use the well known Secure Socket |
|
85 | 85 | Layer (SSL) protocol [RFC5246]_. We use the implementation of this protocol |
|
86 | 86 | provided by the OpenSSL project through the pyOpenSSL [pyOpenSSL]_ Python |
|
87 | 87 | bindings to OpenSSL. |
|
88 | 88 | |
|
89 | 89 | Authentication |
|
90 | 90 | -------------- |
|
91 | 91 | |
|
92 | 92 | IPython clients and engines must also authenticate themselves with the |
|
93 | 93 | controller. This is handled in a capabilities based security model |
|
94 | 94 | [Capability]_. In this model, the controller creates a strong cryptographic |
|
95 | 95 | key or token that represents each set of capability that the controller |
|
96 | 96 | offers. Any party who has this key and presents it to the controller has full |
|
97 | 97 | access to the corresponding capabilities of the controller. This model is |
|
98 | 98 | analogous to using a physical key to gain access to physical items |
|
99 | 99 | (capabilities) behind a locked door. |
|
100 | 100 | |
|
101 | 101 | For a capabilities based authentication system to prevent unauthorized access, |
|
102 | 102 | two things must be ensured: |
|
103 | 103 | |
|
104 | 104 | * The keys must be cryptographically strong. Otherwise attackers could gain |
|
105 | 105 | access by a simple brute force key guessing attack. |
|
106 | 106 | * The actual keys must be distributed only to authorized parties. |
|
107 | 107 | |
|
108 | 108 | The keys in Foolscap are called Foolscap URL's or FURLs. The following section |
|
109 | 109 | gives details about how these FURLs are created in Foolscap. The IPython |
|
110 | 110 | controller creates a number of FURLs for different purposes: |
|
111 | 111 | |
|
112 | 112 | * One FURL that grants IPython engines access to the controller. Also |
|
113 | 113 | implicit in this access is permission to execute code sent by an |
|
114 | 114 | authenticated IPython client. |
|
115 | 115 | * Two or more FURLs that grant IPython clients access to the controller. |
|
116 | 116 | Implicit in this access is permission to give the controller's engine code |
|
117 | 117 | to execute. |
|
118 | 118 | |
|
119 | 119 | Upon starting, the controller creates these different FURLS and writes them |
|
120 | 120 | files in the user-read-only directory :file:`$HOME/.ipython/security`. Thus, |
|
121 | 121 | only the user who starts the controller has access to the FURLs. |
|
122 | 122 | |
|
123 | 123 | For an IPython client or engine to authenticate with a controller, it must |
|
124 | 124 | present the appropriate FURL to the controller upon connecting. If the |
|
125 | 125 | FURL matches what the controller expects for a given capability, access is |
|
126 | 126 | granted. If not, access is denied. The exchange of FURLs is done after |
|
127 | 127 | encrypted communications channels have been established to prevent attackers |
|
128 | 128 | from capturing them. |
|
129 | 129 | |
|
130 | 130 | .. note:: |
|
131 | 131 | |
|
132 | 132 | The FURL is similar to an unsigned private key in SSH. |
|
133 | 133 | |
|
134 | 134 | Details of the Foolscap handshake |
|
135 | 135 | --------------------------------- |
|
136 | 136 | |
|
137 | 137 | In this section we detail the precise security handshake that takes place at |
|
138 | 138 | the beginning of any network connection in IPython. For the purposes of this |
|
139 | 139 | discussion, the SERVER is the IPython controller process and the CLIENT is the |
|
140 | 140 | IPython engine or client process. |
|
141 | 141 | |
|
142 | 142 | Upon starting, all IPython processes do the following: |
|
143 | 143 | |
|
144 | 144 | 1. Create a public key x509 certificate (ISO/IEC 9594). |
|
145 | 145 | 2. Create a hash of the contents of the certificate using the SHA-1 algorithm. |
|
146 | 146 | The base-32 encoded version of this hash is saved by the process as its |
|
147 | 147 | process id (actually in Foolscap, this is the Tub id, but here refer to |
|
148 | 148 | it as the process id). |
|
149 | 149 | |
|
150 | 150 | Upon starting, the IPython controller also does the following: |
|
151 | 151 | |
|
152 | 152 | 1. Save the x509 certificate to disk in a secure location. The CLIENT |
|
153 | 153 | certificate is never saved to disk. |
|
154 | 154 | 2. Create a FURL for each capability that the controller has. There are |
|
155 | 155 | separate capabilities the controller offers for clients and engines. The |
|
156 | 156 | FURL is created using: a) the process id of the SERVER, b) the IP |
|
157 | 157 | address and port the SERVER is listening on and c) a 160 bit, |
|
158 | 158 | cryptographically secure string that represents the capability (the |
|
159 | 159 | "capability id"). |
|
160 | 160 | 3. The FURLs are saved to disk in a secure location on the SERVER's host. |
|
161 | 161 | |
|
162 | 162 | For a CLIENT to be able to connect to the SERVER and access a capability of |
|
163 | 163 | that SERVER, the CLIENT must have knowledge of the FURL for that SERVER's |
|
164 | 164 | capability. This typically requires that the file containing the FURL be |
|
165 | 165 | moved from the SERVER's host to the CLIENT's host. This is done by the end |
|
166 | 166 | user who started the SERVER and wishes to have a CLIENT connect to the SERVER. |
|
167 | 167 | |
|
168 | 168 | When a CLIENT connects to the SERVER, the following handshake protocol takes |
|
169 | 169 | place: |
|
170 | 170 | |
|
171 | 171 | 1. The CLIENT tells the SERVER what process (or Tub) id it expects the SERVER |
|
172 | 172 | to have. |
|
173 | 173 | 2. If the SERVER has that process id, it notifies the CLIENT that it will now |
|
174 | 174 | enter encrypted mode. If the SERVER has a different id, the SERVER aborts. |
|
175 | 175 | 3. Both CLIENT and SERVER initiate the SSL handshake protocol. |
|
176 | 176 | 4. Both CLIENT and SERVER request the certificate of their peer and verify |
|
177 | 177 | that certificate. If this succeeds, all further communications are |
|
178 | 178 | encrypted. |
|
179 | 179 | 5. Both CLIENT and SERVER send a hello block containing connection parameters |
|
180 | 180 | and their process id. |
|
181 | 181 | 6. The CLIENT and SERVER check that their peer's stated process id matches the |
|
182 | 182 | hash of the x509 certificate the peer presented. If not, the connection is |
|
183 | 183 | aborted. |
|
184 | 184 | 7. The CLIENT verifies that the SERVER's stated id matches the id of the |
|
185 | 185 | SERVER the CLIENT is intending to connect to. If not, the connection is |
|
186 | 186 | aborted. |
|
187 | 187 | 8. The CLIENT and SERVER elect a master who decides on the final connection |
|
188 | 188 | parameters. |
|
189 | 189 | |
|
190 | 190 | The public/private key pair associated with each process's x509 certificate |
|
191 | 191 | are completely hidden from this handshake protocol. There are however, used |
|
192 | 192 | internally by OpenSSL as part of the SSL handshake protocol. Each process |
|
193 | 193 | keeps their own private key hidden and sends its peer only the public key |
|
194 | 194 | (embedded in the certificate). |
|
195 | 195 | |
|
196 | 196 | Finally, when the CLIENT requests access to a particular SERVER capability, |
|
197 | 197 | the following happens: |
|
198 | 198 | |
|
199 | 199 | 1. The CLIENT asks the SERVER for access to a capability by presenting that |
|
200 | 200 | capabilities id. |
|
201 | 201 | 2. If the SERVER has a capability with that id, access is granted. If not, |
|
202 | 202 | access is not granted. |
|
203 | 203 | 3. Once access has been gained, the CLIENT can use the capability. |
|
204 | 204 | |
|
205 | 205 | Specific security vulnerabilities |
|
206 | 206 | ================================= |
|
207 | 207 | |
|
208 | 208 | There are a number of potential security vulnerabilities present in IPython's |
|
209 | 209 | architecture. In this section we discuss those vulnerabilities and detail how |
|
210 | 210 | the security architecture described above prevents them from being exploited. |
|
211 | 211 | |
|
212 | 212 | Unauthorized clients |
|
213 | 213 | -------------------- |
|
214 | 214 | |
|
215 | 215 | The IPython client can instruct the IPython engines to execute arbitrary |
|
216 | 216 | Python code with the permissions of the user who started the engines. If an |
|
217 | 217 | attacker were able to connect their own hostile IPython client to the IPython |
|
218 | 218 | controller, they could instruct the engines to execute code. |
|
219 | 219 | |
|
220 | 220 | This attack is prevented by the capabilities based client authentication |
|
221 | 221 | performed after the encrypted channel has been established. The relevant |
|
222 | 222 | authentication information is encoded into the FURL that clients must |
|
223 | 223 | present to gain access to the IPython controller. By limiting the distribution |
|
224 | 224 | of those FURLs, a user can grant access to only authorized persons. |
|
225 | 225 | |
|
226 | 226 | It is highly unlikely that a client FURL could be guessed by an attacker |
|
227 | 227 | in a brute force guessing attack. A given instance of the IPython controller |
|
228 | 228 | only runs for a relatively short amount of time (on the order of hours). Thus |
|
229 | 229 | an attacker would have only a limited amount of time to test a search space of |
|
230 | 230 | size 2**320. Furthermore, even if a controller were to run for a longer amount |
|
231 | 231 | of time, this search space is quite large (larger for instance than that of |
|
232 | 232 | typical username/password pair). |
|
233 | 233 | |
|
234 | 234 | Unauthorized engines |
|
235 | 235 | -------------------- |
|
236 | 236 | |
|
237 | 237 | If an attacker were able to connect a hostile engine to a user's controller, |
|
238 | 238 | the user might unknowingly send sensitive code or data to the hostile engine. |
|
239 | 239 | This attacker's engine would then have full access to that code and data. |
|
240 | 240 | |
|
241 | 241 | This type of attack is prevented in the same way as the unauthorized client |
|
242 | 242 | attack, through the usage of the capabilities based authentication scheme. |
|
243 | 243 | |
|
244 | 244 | Unauthorized controllers |
|
245 | 245 | ------------------------ |
|
246 | 246 | |
|
247 | 247 | It is also possible that an attacker could try to convince a user's IPython |
|
248 | 248 | client or engine to connect to a hostile IPython controller. That controller |
|
249 | 249 | would then have full access to the code and data sent between the IPython |
|
250 | 250 | client and the IPython engines. |
|
251 | 251 | |
|
252 | 252 | Again, this attack is prevented through the FURLs, which ensure that a |
|
253 | 253 | client or engine connects to the correct controller. It is also important to |
|
254 | 254 | note that the FURLs also encode the IP address and port that the |
|
255 | 255 | controller is listening on, so there is little chance of mistakenly connecting |
|
256 | 256 | to a controller running on a different IP address and port. |
|
257 | 257 | |
|
258 | 258 | When starting an engine or client, a user must specify which FURL to use |
|
259 | 259 | for that connection. Thus, in order to introduce a hostile controller, the |
|
260 | 260 | attacker must convince the user to use the FURLs associated with the |
|
261 | 261 | hostile controller. As long as a user is diligent in only using FURLs from |
|
262 | 262 | trusted sources, this attack is not possible. |
|
263 | 263 | |
|
264 | 264 | Other security measures |
|
265 | 265 | ======================= |
|
266 | 266 | |
|
267 | 267 | A number of other measures are taken to further limit the security risks |
|
268 | 268 | involved in running the IPython kernel. |
|
269 | 269 | |
|
270 | 270 | First, by default, the IPython controller listens on random port numbers. |
|
271 | 271 | While this can be overridden by the user, in the default configuration, an |
|
272 | 272 | attacker would have to do a port scan to even find a controller to attack. |
|
273 | 273 | When coupled with the relatively short running time of a typical controller |
|
274 | 274 | (on the order of hours), an attacker would have to work extremely hard and |
|
275 | 275 | extremely *fast* to even find a running controller to attack. |
|
276 | 276 | |
|
277 | 277 | Second, much of the time, especially when run on supercomputers or clusters, |
|
278 | 278 | the controller is running behind a firewall. Thus, for engines or client to |
|
279 | 279 | connect to the controller: |
|
280 | 280 | |
|
281 | 281 | * The different processes have to all be behind the firewall. |
|
282 | 282 | |
|
283 | 283 | or: |
|
284 | 284 | |
|
285 | 285 | * The user has to use SSH port forwarding to tunnel the |
|
286 | 286 | connections through the firewall. |
|
287 | 287 | |
|
288 | 288 | In either case, an attacker is presented with addition barriers that prevent |
|
289 | 289 | attacking or even probing the system. |
|
290 | 290 | |
|
291 | 291 | Summary |
|
292 | 292 | ======= |
|
293 | 293 | |
|
294 | 294 | IPython's architecture has been carefully designed with security in mind. The |
|
295 | 295 | capabilities based authentication model, in conjunction with the encrypted |
|
296 | 296 | TCP/IP channels, address the core potential vulnerabilities in the system, |
|
297 | 297 | while still enabling user's to use the system in open networks. |
|
298 | 298 | |
|
299 | 299 | Other questions |
|
300 | 300 | =============== |
|
301 | 301 | |
|
302 | 302 | About keys |
|
303 | 303 | ---------- |
|
304 | 304 | |
|
305 | 305 | Can you clarify the roles of the certificate and its keys versus the FURL, |
|
306 | 306 | which is also called a key? |
|
307 | 307 | |
|
308 | 308 | The certificate created by IPython processes is a standard public key x509 |
|
309 | 309 | certificate, that is used by the SSL handshake protocol to setup encrypted |
|
310 | 310 | channel between the controller and the IPython engine or client. This public |
|
311 | 311 | and private key associated with this certificate are used only by the SSL |
|
312 | 312 | handshake protocol in setting up this encrypted channel. |
|
313 | 313 | |
|
314 | 314 | The FURL serves a completely different and independent purpose from the |
|
315 | 315 | key pair associated with the certificate. When we refer to a FURL as a |
|
316 | 316 | key, we are using the word "key" in the capabilities based security model |
|
317 | 317 | sense. This has nothing to do with "key" in the public/private key sense used |
|
318 | 318 | in the SSL protocol. |
|
319 | 319 | |
|
320 | 320 | With that said the FURL is used as an cryptographic key, to grant |
|
321 | 321 | IPython engines and clients access to particular capabilities that the |
|
322 | 322 | controller offers. |
|
323 | 323 | |
|
324 | 324 | Self signed certificates |
|
325 | 325 | ------------------------ |
|
326 | 326 | |
|
327 | 327 | Is the controller creating a self-signed certificate? Is this created for per |
|
328 | 328 | instance/session, one-time-setup or each-time the controller is started? |
|
329 | 329 | |
|
330 | 330 | The Foolscap network protocol, which handles the SSL protocol details, creates |
|
331 | 331 | a self-signed x509 certificate using OpenSSL for each IPython process. The |
|
332 | 332 | lifetime of the certificate is handled differently for the IPython controller |
|
333 | 333 | and the engines/client. |
|
334 | 334 | |
|
335 | 335 | For the IPython engines and client, the certificate is only held in memory for |
|
336 | 336 | the lifetime of its process. It is never written to disk. |
|
337 | 337 | |
|
338 | 338 | For the controller, the certificate can be created anew each time the |
|
339 | 339 | controller starts or it can be created once and reused each time the |
|
340 | 340 | controller starts. If at any point, the certificate is deleted, a new one is |
|
341 | 341 | created the next time the controller starts. |
|
342 | 342 | |
|
343 | 343 | SSL private key |
|
344 | 344 | --------------- |
|
345 | 345 | |
|
346 | 346 | How the private key (associated with the certificate) is distributed? |
|
347 | 347 | |
|
348 | 348 | In the usual implementation of the SSL protocol, the private key is never |
|
349 | 349 | distributed. We follow this standard always. |
|
350 | 350 | |
|
351 | 351 | SSL versus Foolscap authentication |
|
352 | 352 | ---------------------------------- |
|
353 | 353 | |
|
354 | 354 | Many SSL connections only perform one sided authentication (the server to the |
|
355 | 355 | client). How is the client authentication in IPython's system related to SSL |
|
356 | 356 | authentication? |
|
357 | 357 | |
|
358 | 358 | We perform a two way SSL handshake in which both parties request and verify |
|
359 | 359 | the certificate of their peer. This mutual authentication is handled by the |
|
360 | 360 | SSL handshake and is separate and independent from the additional |
|
361 | 361 | authentication steps that the CLIENT and SERVER perform after an encrypted |
|
362 | 362 | channel is established. |
|
363 | 363 | |
|
364 | 364 | .. [RFC5246] <http://tools.ietf.org/html/rfc5246> |
|
365 | ||
|
366 |
@@ -1,120 +1,121 b'' | |||
|
1 | 1 | .. _paralleltask: |
|
2 | 2 | |
|
3 | 3 | ========================== |
|
4 | 4 | The IPython task interface |
|
5 | 5 | ========================== |
|
6 | 6 | |
|
7 | 7 | The task interface to the controller presents the engines as a fault tolerant, |
|
8 | 8 | dynamic load-balanced system or workers. Unlike the multiengine interface, in |
|
9 | 9 | the task interface, the user have no direct access to individual engines. In |
|
10 | 10 | some ways, this interface is simpler, but in other ways it is more powerful. |
|
11 | 11 | |
|
12 | 12 | Best of all the user can use both of these interfaces running at the same time |
|
13 | 13 | to take advantage or both of their strengths. When the user can break up the |
|
14 | 14 | user's work into segments that do not depend on previous execution, the task |
|
15 | 15 | interface is ideal. But it also has more power and flexibility, allowing the |
|
16 | 16 | user to guide the distribution of jobs, without having to assign tasks to |
|
17 | 17 | engines explicitly. |
|
18 | 18 | |
|
19 | 19 | Starting the IPython controller and engines |
|
20 | 20 | =========================================== |
|
21 | 21 | |
|
22 | 22 | To follow along with this tutorial, you will need to start the IPython |
|
23 | 23 | controller and four IPython engines. The simplest way of doing this is to use |
|
24 | 24 | the :command:`ipcluster` command:: |
|
25 | 25 | |
|
26 | 26 | $ ipcluster local -n 4 |
|
27 | 27 | |
|
28 | 28 | For more detailed information about starting the controller and engines, see |
|
29 | 29 | our :ref:`introduction <ip1par>` to using IPython for parallel computing. |
|
30 | 30 | |
|
31 | 31 | Creating a ``TaskClient`` instance |
|
32 | 32 | ========================================= |
|
33 | 33 | |
|
34 | 34 | The first step is to import the IPython :mod:`IPython.kernel.client` module |
|
35 | 35 | and then create a :class:`TaskClient` instance: |
|
36 | 36 | |
|
37 | 37 | .. sourcecode:: ipython |
|
38 | 38 | |
|
39 | 39 | In [1]: from IPython.kernel import client |
|
40 | 40 | |
|
41 | 41 | In [2]: tc = client.TaskClient() |
|
42 | 42 | |
|
43 | 43 | This form assumes that the :file:`ipcontroller-tc.furl` is in the |
|
44 | 44 | :file:`~./ipython/security` directory on the client's host. If not, the |
|
45 | 45 | location of the FURL file must be given as an argument to the |
|
46 | 46 | constructor: |
|
47 | 47 | |
|
48 | 48 | .. sourcecode:: ipython |
|
49 | 49 | |
|
50 | 50 | In [2]: mec = client.TaskClient('/path/to/my/ipcontroller-tc.furl') |
|
51 | 51 | |
|
52 | 52 | Quick and easy parallelism |
|
53 | 53 | ========================== |
|
54 | 54 | |
|
55 | 55 | In many cases, you simply want to apply a Python function to a sequence of |
|
56 | 56 | objects, but *in parallel*. Like the multiengine interface, the task interface |
|
57 | 57 | provides two simple ways of accomplishing this: a parallel version of |
|
58 | 58 | :func:`map` and ``@parallel`` function decorator. However, the verions in the |
|
59 | 59 | task interface have one important difference: they are dynamically load |
|
60 | 60 | balanced. Thus, if the execution time per item varies significantly, you |
|
61 | 61 | should use the versions in the task interface. |
|
62 | 62 | |
|
63 | 63 | Parallel map |
|
64 | 64 | ------------ |
|
65 | 65 | |
|
66 | 66 | The parallel :meth:`map` in the task interface is similar to that in the |
|
67 | 67 | multiengine interface: |
|
68 | 68 | |
|
69 | 69 | .. sourcecode:: ipython |
|
70 | 70 | |
|
71 | 71 | In [63]: serial_result = map(lambda x:x**10, range(32)) |
|
72 | 72 | |
|
73 | 73 | In [64]: parallel_result = tc.map(lambda x:x**10, range(32)) |
|
74 | 74 | |
|
75 | 75 | In [65]: serial_result==parallel_result |
|
76 | 76 | Out[65]: True |
|
77 | 77 | |
|
78 | 78 | Parallel function decorator |
|
79 | 79 | --------------------------- |
|
80 | 80 | |
|
81 | 81 | Parallel functions are just like normal function, but they can be called on |
|
82 | 82 | sequences and *in parallel*. The multiengine interface provides a decorator |
|
83 | 83 | that turns any Python function into a parallel function: |
|
84 | 84 | |
|
85 | 85 | .. sourcecode:: ipython |
|
86 | 86 | |
|
87 | 87 | In [10]: @tc.parallel() |
|
88 | 88 | ....: def f(x): |
|
89 | 89 | ....: return 10.0*x**4 |
|
90 | 90 | ....: |
|
91 | 91 | |
|
92 | 92 | In [11]: f(range(32)) # this is done in parallel |
|
93 | 93 | Out[11]: |
|
94 | 94 | [0.0,10.0,160.0,...] |
|
95 | 95 | |
|
96 | 96 | More details |
|
97 | 97 | ============ |
|
98 | 98 | |
|
99 | 99 | The :class:`TaskClient` has many more powerful features that allow quite a bit |
|
100 | 100 | of flexibility in how tasks are defined and run. The next places to look are |
|
101 | 101 | in the following classes: |
|
102 | 102 | |
|
103 | 103 | * :class:`IPython.kernel.client.TaskClient` |
|
104 | 104 | * :class:`IPython.kernel.client.StringTask` |
|
105 | 105 | * :class:`IPython.kernel.client.MapTask` |
|
106 | 106 | |
|
107 | 107 | The following is an overview of how to use these classes together: |
|
108 | 108 | |
|
109 | 109 | 1. Create a :class:`TaskClient`. |
|
110 | 110 | 2. Create one or more instances of :class:`StringTask` or :class:`MapTask` |
|
111 | 111 | to define your tasks. |
|
112 | 112 | 3. Submit your tasks to using the :meth:`run` method of your |
|
113 | 113 | :class:`TaskClient` instance. |
|
114 | 114 | 4. Use :meth:`TaskClient.get_task_result` to get the results of the |
|
115 | 115 | tasks. |
|
116 | 116 | |
|
117 | 117 | We are in the process of developing more detailed information about the task |
|
118 | 118 | interface. For now, the docstrings of the :class:`TaskClient`, |
|
119 | 119 | :class:`StringTask` and :class:`MapTask` classes should be consulted. |
|
120 | 120 | |
|
121 |
@@ -1,426 +1,427 b'' | |||
|
1 | 1 | """Attempt to generate templates for module reference with Sphinx |
|
2 | 2 | |
|
3 | 3 | XXX - we exclude extension modules |
|
4 | 4 | |
|
5 | 5 | To include extension modules, first identify them as valid in the |
|
6 | 6 | ``_uri2path`` method, then handle them in the ``_parse_module`` script. |
|
7 | 7 | |
|
8 | 8 | We get functions and classes by parsing the text of .py files. |
|
9 | 9 | Alternatively we could import the modules for discovery, and we'd have |
|
10 | 10 | to do that for extension modules. This would involve changing the |
|
11 | 11 | ``_parse_module`` method to work via import and introspection, and |
|
12 | 12 | might involve changing ``discover_modules`` (which determines which |
|
13 | 13 | files are modules, and therefore which module URIs will be passed to |
|
14 | 14 | ``_parse_module``). |
|
15 | 15 | |
|
16 | 16 | NOTE: this is a modified version of a script originally shipped with the |
|
17 | 17 | PyMVPA project, which we've adapted for NIPY use. PyMVPA is an MIT-licensed |
|
18 | 18 | project.""" |
|
19 | 19 | |
|
20 | 20 | # Stdlib imports |
|
21 | 21 | import os |
|
22 | 22 | import re |
|
23 | 23 | |
|
24 | 24 | # Functions and classes |
|
25 | 25 | class ApiDocWriter(object): |
|
26 | 26 | ''' Class for automatic detection and parsing of API docs |
|
27 | 27 | to Sphinx-parsable reST format''' |
|
28 | 28 | |
|
29 | 29 | # only separating first two levels |
|
30 | 30 | rst_section_levels = ['*', '=', '-', '~', '^'] |
|
31 | 31 | |
|
32 | 32 | def __init__(self, |
|
33 | 33 | package_name, |
|
34 | 34 | rst_extension='.rst', |
|
35 | 35 | package_skip_patterns=None, |
|
36 | 36 | module_skip_patterns=None, |
|
37 | 37 | ): |
|
38 | 38 | ''' Initialize package for parsing |
|
39 | 39 | |
|
40 | 40 | Parameters |
|
41 | 41 | ---------- |
|
42 | 42 | package_name : string |
|
43 | 43 | Name of the top-level package. *package_name* must be the |
|
44 | 44 | name of an importable package |
|
45 | 45 | rst_extension : string, optional |
|
46 | 46 | Extension for reST files, default '.rst' |
|
47 | 47 | package_skip_patterns : None or sequence of {strings, regexps} |
|
48 | 48 | Sequence of strings giving URIs of packages to be excluded |
|
49 | 49 | Operates on the package path, starting at (including) the |
|
50 | 50 | first dot in the package path, after *package_name* - so, |
|
51 | 51 | if *package_name* is ``sphinx``, then ``sphinx.util`` will |
|
52 | 52 | result in ``.util`` being passed for earching by these |
|
53 | 53 | regexps. If is None, gives default. Default is: |
|
54 | 54 | ['\.tests$'] |
|
55 | 55 | module_skip_patterns : None or sequence |
|
56 | 56 | Sequence of strings giving URIs of modules to be excluded |
|
57 | 57 | Operates on the module name including preceding URI path, |
|
58 | 58 | back to the first dot after *package_name*. For example |
|
59 | 59 | ``sphinx.util.console`` results in the string to search of |
|
60 | 60 | ``.util.console`` |
|
61 | 61 | If is None, gives default. Default is: |
|
62 | 62 | ['\.setup$', '\._'] |
|
63 | 63 | ''' |
|
64 | 64 | if package_skip_patterns is None: |
|
65 | 65 | package_skip_patterns = ['\\.tests$'] |
|
66 | 66 | if module_skip_patterns is None: |
|
67 | 67 | module_skip_patterns = ['\\.setup$', '\\._'] |
|
68 | 68 | self.package_name = package_name |
|
69 | 69 | self.rst_extension = rst_extension |
|
70 | 70 | self.package_skip_patterns = package_skip_patterns |
|
71 | 71 | self.module_skip_patterns = module_skip_patterns |
|
72 | 72 | |
|
73 | 73 | def get_package_name(self): |
|
74 | 74 | return self._package_name |
|
75 | 75 | |
|
76 | 76 | def set_package_name(self, package_name): |
|
77 | 77 | ''' Set package_name |
|
78 | 78 | |
|
79 | 79 | >>> docwriter = ApiDocWriter('sphinx') |
|
80 | 80 | >>> import sphinx |
|
81 | 81 | >>> docwriter.root_path == sphinx.__path__[0] |
|
82 | 82 | True |
|
83 | 83 | >>> docwriter.package_name = 'docutils' |
|
84 | 84 | >>> import docutils |
|
85 | 85 | >>> docwriter.root_path == docutils.__path__[0] |
|
86 | 86 | True |
|
87 | 87 | ''' |
|
88 | 88 | # It's also possible to imagine caching the module parsing here |
|
89 | 89 | self._package_name = package_name |
|
90 | 90 | self.root_module = __import__(package_name) |
|
91 | 91 | self.root_path = self.root_module.__path__[0] |
|
92 | 92 | self.written_modules = None |
|
93 | 93 | |
|
94 | 94 | package_name = property(get_package_name, set_package_name, None, |
|
95 | 95 | 'get/set package_name') |
|
96 | 96 | |
|
97 | 97 | def _get_object_name(self, line): |
|
98 | 98 | ''' Get second token in line |
|
99 | 99 | >>> docwriter = ApiDocWriter('sphinx') |
|
100 | 100 | >>> docwriter._get_object_name(" def func(): ") |
|
101 | 101 | 'func' |
|
102 | 102 | >>> docwriter._get_object_name(" class Klass(object): ") |
|
103 | 103 | 'Klass' |
|
104 | 104 | >>> docwriter._get_object_name(" class Klass: ") |
|
105 | 105 | 'Klass' |
|
106 | 106 | ''' |
|
107 | 107 | name = line.split()[1].split('(')[0].strip() |
|
108 | 108 | # in case we have classes which are not derived from object |
|
109 | 109 | # ie. old style classes |
|
110 | 110 | return name.rstrip(':') |
|
111 | 111 | |
|
112 | 112 | def _uri2path(self, uri): |
|
113 | 113 | ''' Convert uri to absolute filepath |
|
114 | 114 | |
|
115 | 115 | Parameters |
|
116 | 116 | ---------- |
|
117 | 117 | uri : string |
|
118 | 118 | URI of python module to return path for |
|
119 | 119 | |
|
120 | 120 | Returns |
|
121 | 121 | ------- |
|
122 | 122 | path : None or string |
|
123 | 123 | Returns None if there is no valid path for this URI |
|
124 | 124 | Otherwise returns absolute file system path for URI |
|
125 | 125 | |
|
126 | 126 | Examples |
|
127 | 127 | -------- |
|
128 | 128 | >>> docwriter = ApiDocWriter('sphinx') |
|
129 | 129 | >>> import sphinx |
|
130 | 130 | >>> modpath = sphinx.__path__[0] |
|
131 | 131 | >>> res = docwriter._uri2path('sphinx.builder') |
|
132 | 132 | >>> res == os.path.join(modpath, 'builder.py') |
|
133 | 133 | True |
|
134 | 134 | >>> res = docwriter._uri2path('sphinx') |
|
135 | 135 | >>> res == os.path.join(modpath, '__init__.py') |
|
136 | 136 | True |
|
137 | 137 | >>> docwriter._uri2path('sphinx.does_not_exist') |
|
138 | 138 | |
|
139 | 139 | ''' |
|
140 | 140 | if uri == self.package_name: |
|
141 | 141 | return os.path.join(self.root_path, '__init__.py') |
|
142 | 142 | path = uri.replace('.', os.path.sep) |
|
143 | 143 | path = path.replace(self.package_name + os.path.sep, '') |
|
144 | 144 | path = os.path.join(self.root_path, path) |
|
145 | 145 | # XXX maybe check for extensions as well? |
|
146 | 146 | if os.path.exists(path + '.py'): # file |
|
147 | 147 | path += '.py' |
|
148 | 148 | elif os.path.exists(os.path.join(path, '__init__.py')): |
|
149 | 149 | path = os.path.join(path, '__init__.py') |
|
150 | 150 | else: |
|
151 | 151 | return None |
|
152 | 152 | return path |
|
153 | 153 | |
|
154 | 154 | def _path2uri(self, dirpath): |
|
155 | 155 | ''' Convert directory path to uri ''' |
|
156 | 156 | relpath = dirpath.replace(self.root_path, self.package_name) |
|
157 | 157 | if relpath.startswith(os.path.sep): |
|
158 | 158 | relpath = relpath[1:] |
|
159 | 159 | return relpath.replace(os.path.sep, '.') |
|
160 | 160 | |
|
161 | 161 | def _parse_module(self, uri): |
|
162 | 162 | ''' Parse module defined in *uri* ''' |
|
163 | 163 | filename = self._uri2path(uri) |
|
164 | 164 | if filename is None: |
|
165 | 165 | # nothing that we could handle here. |
|
166 | 166 | return ([],[]) |
|
167 | 167 | f = open(filename, 'rt') |
|
168 | 168 | functions, classes = self._parse_lines(f) |
|
169 | 169 | f.close() |
|
170 | 170 | return functions, classes |
|
171 | 171 | |
|
172 | 172 | def _parse_lines(self, linesource): |
|
173 | 173 | ''' Parse lines of text for functions and classes ''' |
|
174 | 174 | functions = [] |
|
175 | 175 | classes = [] |
|
176 | 176 | for line in linesource: |
|
177 | 177 | if line.startswith('def ') and line.count('('): |
|
178 | 178 | # exclude private stuff |
|
179 | 179 | name = self._get_object_name(line) |
|
180 | 180 | if not name.startswith('_'): |
|
181 | 181 | functions.append(name) |
|
182 | 182 | elif line.startswith('class '): |
|
183 | 183 | # exclude private stuff |
|
184 | 184 | name = self._get_object_name(line) |
|
185 | 185 | if not name.startswith('_'): |
|
186 | 186 | classes.append(name) |
|
187 | 187 | else: |
|
188 | 188 | pass |
|
189 | 189 | functions.sort() |
|
190 | 190 | classes.sort() |
|
191 | 191 | return functions, classes |
|
192 | 192 | |
|
193 | 193 | def generate_api_doc(self, uri): |
|
194 | 194 | '''Make autodoc documentation template string for a module |
|
195 | 195 | |
|
196 | 196 | Parameters |
|
197 | 197 | ---------- |
|
198 | 198 | uri : string |
|
199 | 199 | python location of module - e.g 'sphinx.builder' |
|
200 | 200 | |
|
201 | 201 | Returns |
|
202 | 202 | ------- |
|
203 | 203 | S : string |
|
204 | 204 | Contents of API doc |
|
205 | 205 | ''' |
|
206 | 206 | # get the names of all classes and functions |
|
207 | 207 | functions, classes = self._parse_module(uri) |
|
208 | 208 | if not len(functions) and not len(classes): |
|
209 | 209 | print 'WARNING: Empty -',uri # dbg |
|
210 | 210 | return '' |
|
211 | 211 | |
|
212 | 212 | # Make a shorter version of the uri that omits the package name for |
|
213 | 213 | # titles |
|
214 | 214 | uri_short = re.sub(r'^%s\.' % self.package_name,'',uri) |
|
215 | 215 | |
|
216 | 216 | ad = '.. AUTO-GENERATED FILE -- DO NOT EDIT!\n\n' |
|
217 | 217 | |
|
218 | 218 | chap_title = uri_short |
|
219 | 219 | ad += (chap_title+'\n'+ self.rst_section_levels[1] * len(chap_title) |
|
220 | 220 | + '\n\n') |
|
221 | 221 | |
|
222 | 222 | # Set the chapter title to read 'module' for all modules except for the |
|
223 | 223 | # main packages |
|
224 | 224 | if '.' in uri: |
|
225 | 225 | title = 'Module: :mod:`' + uri_short + '`' |
|
226 | 226 | else: |
|
227 | 227 | title = ':mod:`' + uri_short + '`' |
|
228 | 228 | ad += title + '\n' + self.rst_section_levels[2] * len(title) |
|
229 | 229 | |
|
230 | 230 | if len(classes): |
|
231 | 231 | ad += '\nInheritance diagram for ``%s``:\n\n' % uri |
|
232 | 232 | ad += '.. inheritance-diagram:: %s \n' % uri |
|
233 | 233 | ad += ' :parts: 3\n' |
|
234 | 234 | |
|
235 | 235 | ad += '\n.. automodule:: ' + uri + '\n' |
|
236 | 236 | ad += '\n.. currentmodule:: ' + uri + '\n' |
|
237 | 237 | multi_class = len(classes) > 1 |
|
238 | 238 | multi_fx = len(functions) > 1 |
|
239 | 239 | if multi_class: |
|
240 | 240 | ad += '\n' + 'Classes' + '\n' + \ |
|
241 | 241 | self.rst_section_levels[2] * 7 + '\n' |
|
242 | 242 | elif len(classes) and multi_fx: |
|
243 | 243 | ad += '\n' + 'Class' + '\n' + \ |
|
244 | 244 | self.rst_section_levels[2] * 5 + '\n' |
|
245 | 245 | for c in classes: |
|
246 | 246 | ad += '\n:class:`' + c + '`\n' \ |
|
247 | 247 | + self.rst_section_levels[multi_class + 2 ] * \ |
|
248 | 248 | (len(c)+9) + '\n\n' |
|
249 | 249 | ad += '\n.. autoclass:: ' + c + '\n' |
|
250 | 250 | # must NOT exclude from index to keep cross-refs working |
|
251 | 251 | ad += ' :members:\n' \ |
|
252 | 252 | ' :undoc-members:\n' \ |
|
253 | 253 | ' :show-inheritance:\n' \ |
|
254 | ' :inherited-members:\n' \ | |
|
254 | 255 | '\n' \ |
|
255 | 256 | ' .. automethod:: __init__\n' |
|
256 | 257 | if multi_fx: |
|
257 | 258 | ad += '\n' + 'Functions' + '\n' + \ |
|
258 | 259 | self.rst_section_levels[2] * 9 + '\n\n' |
|
259 | 260 | elif len(functions) and multi_class: |
|
260 | 261 | ad += '\n' + 'Function' + '\n' + \ |
|
261 | 262 | self.rst_section_levels[2] * 8 + '\n\n' |
|
262 | 263 | for f in functions: |
|
263 | 264 | # must NOT exclude from index to keep cross-refs working |
|
264 | 265 | ad += '\n.. autofunction:: ' + uri + '.' + f + '\n\n' |
|
265 | 266 | return ad |
|
266 | 267 | |
|
267 | 268 | def _survives_exclude(self, matchstr, match_type): |
|
268 | 269 | ''' Returns True if *matchstr* does not match patterns |
|
269 | 270 | |
|
270 | 271 | ``self.package_name`` removed from front of string if present |
|
271 | 272 | |
|
272 | 273 | Examples |
|
273 | 274 | -------- |
|
274 | 275 | >>> dw = ApiDocWriter('sphinx') |
|
275 | 276 | >>> dw._survives_exclude('sphinx.okpkg', 'package') |
|
276 | 277 | True |
|
277 | 278 | >>> dw.package_skip_patterns.append('^\\.badpkg$') |
|
278 | 279 | >>> dw._survives_exclude('sphinx.badpkg', 'package') |
|
279 | 280 | False |
|
280 | 281 | >>> dw._survives_exclude('sphinx.badpkg', 'module') |
|
281 | 282 | True |
|
282 | 283 | >>> dw._survives_exclude('sphinx.badmod', 'module') |
|
283 | 284 | True |
|
284 | 285 | >>> dw.module_skip_patterns.append('^\\.badmod$') |
|
285 | 286 | >>> dw._survives_exclude('sphinx.badmod', 'module') |
|
286 | 287 | False |
|
287 | 288 | ''' |
|
288 | 289 | if match_type == 'module': |
|
289 | 290 | patterns = self.module_skip_patterns |
|
290 | 291 | elif match_type == 'package': |
|
291 | 292 | patterns = self.package_skip_patterns |
|
292 | 293 | else: |
|
293 | 294 | raise ValueError('Cannot interpret match type "%s"' |
|
294 | 295 | % match_type) |
|
295 | 296 | # Match to URI without package name |
|
296 | 297 | L = len(self.package_name) |
|
297 | 298 | if matchstr[:L] == self.package_name: |
|
298 | 299 | matchstr = matchstr[L:] |
|
299 | 300 | for pat in patterns: |
|
300 | 301 | try: |
|
301 | 302 | pat.search |
|
302 | 303 | except AttributeError: |
|
303 | 304 | pat = re.compile(pat) |
|
304 | 305 | if pat.search(matchstr): |
|
305 | 306 | return False |
|
306 | 307 | return True |
|
307 | 308 | |
|
308 | 309 | def discover_modules(self): |
|
309 | 310 | ''' Return module sequence discovered from ``self.package_name`` |
|
310 | 311 | |
|
311 | 312 | |
|
312 | 313 | Parameters |
|
313 | 314 | ---------- |
|
314 | 315 | None |
|
315 | 316 | |
|
316 | 317 | Returns |
|
317 | 318 | ------- |
|
318 | 319 | mods : sequence |
|
319 | 320 | Sequence of module names within ``self.package_name`` |
|
320 | 321 | |
|
321 | 322 | Examples |
|
322 | 323 | -------- |
|
323 | 324 | >>> dw = ApiDocWriter('sphinx') |
|
324 | 325 | >>> mods = dw.discover_modules() |
|
325 | 326 | >>> 'sphinx.util' in mods |
|
326 | 327 | True |
|
327 | 328 | >>> dw.package_skip_patterns.append('\.util$') |
|
328 | 329 | >>> 'sphinx.util' in dw.discover_modules() |
|
329 | 330 | False |
|
330 | 331 | >>> |
|
331 | 332 | ''' |
|
332 | 333 | modules = [self.package_name] |
|
333 | 334 | # raw directory parsing |
|
334 | 335 | for dirpath, dirnames, filenames in os.walk(self.root_path): |
|
335 | 336 | # Check directory names for packages |
|
336 | 337 | root_uri = self._path2uri(os.path.join(self.root_path, |
|
337 | 338 | dirpath)) |
|
338 | 339 | for dirname in dirnames[:]: # copy list - we modify inplace |
|
339 | 340 | package_uri = '.'.join((root_uri, dirname)) |
|
340 | 341 | if (self._uri2path(package_uri) and |
|
341 | 342 | self._survives_exclude(package_uri, 'package')): |
|
342 | 343 | modules.append(package_uri) |
|
343 | 344 | else: |
|
344 | 345 | dirnames.remove(dirname) |
|
345 | 346 | # Check filenames for modules |
|
346 | 347 | for filename in filenames: |
|
347 | 348 | module_name = filename[:-3] |
|
348 | 349 | module_uri = '.'.join((root_uri, module_name)) |
|
349 | 350 | if (self._uri2path(module_uri) and |
|
350 | 351 | self._survives_exclude(module_uri, 'module')): |
|
351 | 352 | modules.append(module_uri) |
|
352 | 353 | return sorted(modules) |
|
353 | 354 | |
|
354 | 355 | def write_modules_api(self, modules,outdir): |
|
355 | 356 | # write the list |
|
356 | 357 | written_modules = [] |
|
357 | 358 | for m in modules: |
|
358 | 359 | api_str = self.generate_api_doc(m) |
|
359 | 360 | if not api_str: |
|
360 | 361 | continue |
|
361 | 362 | # write out to file |
|
362 | 363 | outfile = os.path.join(outdir, |
|
363 | 364 | m + self.rst_extension) |
|
364 | 365 | fileobj = open(outfile, 'wt') |
|
365 | 366 | fileobj.write(api_str) |
|
366 | 367 | fileobj.close() |
|
367 | 368 | written_modules.append(m) |
|
368 | 369 | self.written_modules = written_modules |
|
369 | 370 | |
|
370 | 371 | def write_api_docs(self, outdir): |
|
371 | 372 | """Generate API reST files. |
|
372 | 373 | |
|
373 | 374 | Parameters |
|
374 | 375 | ---------- |
|
375 | 376 | outdir : string |
|
376 | 377 | Directory name in which to store files |
|
377 | 378 | We create automatic filenames for each module |
|
378 | 379 | |
|
379 | 380 | Returns |
|
380 | 381 | ------- |
|
381 | 382 | None |
|
382 | 383 | |
|
383 | 384 | Notes |
|
384 | 385 | ----- |
|
385 | 386 | Sets self.written_modules to list of written modules |
|
386 | 387 | """ |
|
387 | 388 | if not os.path.exists(outdir): |
|
388 | 389 | os.mkdir(outdir) |
|
389 | 390 | # compose list of modules |
|
390 | 391 | modules = self.discover_modules() |
|
391 | 392 | self.write_modules_api(modules,outdir) |
|
392 | 393 | |
|
393 | 394 | def write_index(self, outdir, froot='gen', relative_to=None): |
|
394 | 395 | """Make a reST API index file from written files |
|
395 | 396 | |
|
396 | 397 | Parameters |
|
397 | 398 | ---------- |
|
398 | 399 | path : string |
|
399 | 400 | Filename to write index to |
|
400 | 401 | outdir : string |
|
401 | 402 | Directory to which to write generated index file |
|
402 | 403 | froot : string, optional |
|
403 | 404 | root (filename without extension) of filename to write to |
|
404 | 405 | Defaults to 'gen'. We add ``self.rst_extension``. |
|
405 | 406 | relative_to : string |
|
406 | 407 | path to which written filenames are relative. This |
|
407 | 408 | component of the written file path will be removed from |
|
408 | 409 | outdir, in the generated index. Default is None, meaning, |
|
409 | 410 | leave path as it is. |
|
410 | 411 | """ |
|
411 | 412 | if self.written_modules is None: |
|
412 | 413 | raise ValueError('No modules written') |
|
413 | 414 | # Get full filename path |
|
414 | 415 | path = os.path.join(outdir, froot+self.rst_extension) |
|
415 | 416 | # Path written into index is relative to rootpath |
|
416 | 417 | if relative_to is not None: |
|
417 | 418 | relpath = outdir.replace(relative_to + os.path.sep, '') |
|
418 | 419 | else: |
|
419 | 420 | relpath = outdir |
|
420 | 421 | idx = open(path,'wt') |
|
421 | 422 | w = idx.write |
|
422 | 423 | w('.. AUTO-GENERATED FILE -- DO NOT EDIT!\n\n') |
|
423 | 424 | w('.. toctree::\n\n') |
|
424 | 425 | for f in self.written_modules: |
|
425 | 426 | w(' %s\n' % os.path.join(relpath,f)) |
|
426 | 427 | idx.close() |
@@ -1,98 +1,114 b'' | |||
|
1 | 1 | """reST directive for syntax-highlighting ipython interactive sessions. |
|
2 | ||
|
3 | XXX - See what improvements can be made based on the new (as of Sept 2009) | |
|
4 | 'pycon' lexer for the python console. At the very least it will give better | |
|
5 | highlighted tracebacks. | |
|
2 | 6 | """ |
|
3 | 7 | |
|
4 | 8 | #----------------------------------------------------------------------------- |
|
5 | 9 | # Needed modules |
|
6 | 10 | |
|
7 | 11 | # Standard library |
|
8 | 12 | import re |
|
9 | 13 | |
|
10 | 14 | # Third party |
|
11 | 15 | from pygments.lexer import Lexer, do_insertions |
|
12 | 16 | from pygments.lexers.agile import (PythonConsoleLexer, PythonLexer, |
|
13 | 17 | PythonTracebackLexer) |
|
14 | 18 | from pygments.token import Comment, Generic |
|
15 | 19 | |
|
16 | 20 | from sphinx import highlighting |
|
17 | 21 | |
|
18 | ||
|
19 | 22 | #----------------------------------------------------------------------------- |
|
20 | 23 | # Global constants |
|
21 | 24 | line_re = re.compile('.*?\n') |
|
22 | 25 | |
|
23 | 26 | #----------------------------------------------------------------------------- |
|
24 | 27 | # Code begins - classes and functions |
|
25 | 28 | |
|
26 | 29 | class IPythonConsoleLexer(Lexer): |
|
27 | 30 | """ |
|
28 | 31 | For IPython console output or doctests, such as: |
|
29 | 32 | |
|
30 | 33 | .. sourcecode:: ipython |
|
31 | 34 | |
|
32 | 35 | In [1]: a = 'foo' |
|
33 | 36 | |
|
34 | 37 | In [2]: a |
|
35 | 38 | Out[2]: 'foo' |
|
36 | 39 | |
|
37 | 40 | In [3]: print a |
|
38 | 41 | foo |
|
39 | 42 | |
|
40 | 43 | In [4]: 1 / 0 |
|
41 | 44 | |
|
42 | 45 | Notes: |
|
43 | 46 | |
|
44 | 47 | - Tracebacks are not currently supported. |
|
45 | 48 | |
|
46 | 49 | - It assumes the default IPython prompts, not customized ones. |
|
47 | 50 | """ |
|
48 | 51 | |
|
49 | 52 | name = 'IPython console session' |
|
50 | 53 | aliases = ['ipython'] |
|
51 | 54 | mimetypes = ['text/x-ipython-console'] |
|
52 | 55 | input_prompt = re.compile("(In \[[0-9]+\]: )|( \.\.\.+:)") |
|
53 | 56 | output_prompt = re.compile("(Out\[[0-9]+\]: )|( \.\.\.+:)") |
|
54 | 57 | continue_prompt = re.compile(" \.\.\.+:") |
|
55 | 58 | tb_start = re.compile("\-+") |
|
56 | 59 | |
|
57 | 60 | def get_tokens_unprocessed(self, text): |
|
58 | 61 | pylexer = PythonLexer(**self.options) |
|
59 | 62 | tblexer = PythonTracebackLexer(**self.options) |
|
60 | 63 | |
|
61 | 64 | curcode = '' |
|
62 | 65 | insertions = [] |
|
63 | 66 | for match in line_re.finditer(text): |
|
64 | 67 | line = match.group() |
|
65 | 68 | input_prompt = self.input_prompt.match(line) |
|
66 | 69 | continue_prompt = self.continue_prompt.match(line.rstrip()) |
|
67 | 70 | output_prompt = self.output_prompt.match(line) |
|
68 | 71 | if line.startswith("#"): |
|
69 | 72 | insertions.append((len(curcode), |
|
70 | 73 | [(0, Comment, line)])) |
|
71 | 74 | elif input_prompt is not None: |
|
72 | 75 | insertions.append((len(curcode), |
|
73 | 76 | [(0, Generic.Prompt, input_prompt.group())])) |
|
74 | 77 | curcode += line[input_prompt.end():] |
|
75 | 78 | elif continue_prompt is not None: |
|
76 | 79 | insertions.append((len(curcode), |
|
77 | 80 | [(0, Generic.Prompt, continue_prompt.group())])) |
|
78 | 81 | curcode += line[continue_prompt.end():] |
|
79 | 82 | elif output_prompt is not None: |
|
83 | # Use the 'error' token for output. We should probably make | |
|
84 | # our own token, but error is typicaly in a bright color like | |
|
85 | # red, so it works fine for our output prompts. | |
|
80 | 86 | insertions.append((len(curcode), |
|
81 |
[(0, Generic. |
|
|
87 | [(0, Generic.Error, output_prompt.group())])) | |
|
82 | 88 | curcode += line[output_prompt.end():] |
|
83 | 89 | else: |
|
84 | 90 | if curcode: |
|
85 | 91 | for item in do_insertions(insertions, |
|
86 | 92 | pylexer.get_tokens_unprocessed(curcode)): |
|
87 | 93 | yield item |
|
88 | 94 | curcode = '' |
|
89 | 95 | insertions = [] |
|
90 | 96 | yield match.start(), Generic.Output, line |
|
91 | 97 | if curcode: |
|
92 | 98 | for item in do_insertions(insertions, |
|
93 | 99 | pylexer.get_tokens_unprocessed(curcode)): |
|
94 | 100 | yield item |
|
95 | 101 | |
|
102 | ||
|
103 | def setup(app): | |
|
104 | """Setup as a sphinx extension.""" | |
|
105 | ||
|
106 | # This is only a lexer, so adding it below to pygments appears sufficient. | |
|
107 | # But if somebody knows that the right API usage should be to do that via | |
|
108 | # sphinx, by all means fix it here. At least having this setup.py | |
|
109 | # suppresses the sphinx warning we'd get without it. | |
|
110 | pass | |
|
111 | ||
|
96 | 112 | #----------------------------------------------------------------------------- |
|
97 | 113 | # Register the extension as a valid pygments lexer |
|
98 | 114 | highlighting.lexers['ipython'] = IPythonConsoleLexer() |
|
1 | NO CONTENT: file was removed | |
This diff has been collapsed as it changes many lines, (589 lines changed) Show them Hide them |
|
1 | NO CONTENT: file was removed |
|
1 | NO CONTENT: file was removed |
General Comments 0
You need to be logged in to leave comments.
Login now