##// END OF EJS Templates
Merge pull request #3802 from ivanov/doc-to-rst...
Matthias Bussonnier -
r11764:79d36b57 merge
parent child Browse files
Show More
@@ -0,0 +1,16 b''
1 .. _config_index:
2
3 ===============================
4 Configuration and customization
5 ===============================
6
7 .. toctree::
8 :maxdepth: 2
9
10 overview
11 extensions/index
12 ipython
13 integrating
14 editors
15 inputtransforms
16 old
@@ -0,0 +1,15 b''
1 ==================================
2 Using IPython for interactive work
3 ==================================
4
5 .. toctree::
6 :maxdepth: 2
7
8 tutorial
9 tips
10 reference
11 shell
12 qtconsole
13 htmlnotebook
14
15
@@ -0,0 +1,25 b''
1 .. _parallel_index:
2
3 ====================================
4 Using IPython for parallel computing
5 ====================================
6
7 .. toctree::
8 :maxdepth: 2
9
10 parallel_intro
11 parallel_process
12 parallel_multiengine
13 magics
14 parallel_task
15 asyncresult
16 parallel_mpi
17 parallel_db
18 parallel_security
19 parallel_winhpc
20 parallel_demos
21 dag_dependencies
22 parallel_details
23 parallel_transition
24
25
@@ -1,138 +1,137 b''
1 1 # Makefile for Sphinx documentation
2 2 #
3 3
4 4 # You can set these variables from the command line.
5 5 SPHINXOPTS =
6 6 SPHINXBUILD = sphinx-build
7 7 PAPER =
8 8 SRCDIR = source
9 9 BUILDDIR = build
10 10
11 11 # Internal variables.
12 12 PAPEROPT_a4 = -D latex_paper_size=a4
13 13 PAPEROPT_letter = -D latex_paper_size=letter
14 14 ALLSPHINXOPTS = -d build/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) $(SRCDIR)
15 15
16 16 .PHONY: help clean html web pickle htmlhelp latex changes linkcheck api
17 17
18 18 default: html
19 19
20 20 help:
21 21 @echo "Please use \`make <target>' where <target> is one of"
22 22 @echo " html to make standalone HTML files"
23 23 @echo " pickle to make pickle files (usable by e.g. sphinx-web)"
24 24 @echo " htmlhelp to make HTML files and a HTML help project"
25 25 @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
26 26 @echo " texinfo to make Texinfo files"
27 27 @echo " info to make Texinfo files and run them through makeinfo"
28 28 @echo " changes to make an overview over all changed/added/deprecated items"
29 29 @echo " linkcheck to check all external links for integrity"
30 30 @echo
31 31 @echo "Compound utility targets:"
32 32 @echo "pdf latex and then runs the PDF generation"
33 33 @echo "all html and pdf"
34 34 @echo "dist all, and then puts the results in dist/"
35 35 @echo "gitwash-update update git workflow from source repo"
36 36
37 37 clean:
38 38 -rm -rf build/* dist/* $(SRCDIR)/api/generated
39 39
40 40 pdf: latex
41 41 cd build/latex && make all-pdf
42 42
43 43 all: html pdf
44 44
45 45 # For final distribution, only build HTML (our pdf is now so large as to be
46 46 # unusable, takes forever to build and just bloats the downloads). We leave
47 47 # them hardlinked at the top-level so users find them easily, though the
48 48 # original build/html dir is left in-place (useful to reload builds while
49 49 # testing).
50 50 dist: html
51 51 rm -rf html
52 52 cp -al build/html .
53 53 @echo "Build finished. Final docs are in html/"
54 54
55 55 html: api
56 56 mkdir -p build/html build/doctrees
57 57 $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) build/html
58 58 @echo
59 59 @echo "Build finished. The HTML pages are in build/html."
60 60
61 61 api: source/api/generated/gen.txt
62 62
63 63 source/api/generated/gen.txt:
64 64 python autogen_api.py
65 65 @echo "Build API docs finished."
66 66
67 67 pickle:
68 68 mkdir -p build/pickle build/doctrees
69 69 $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) build/pickle
70 70 @echo
71 71 @echo "Build finished; now you can process the pickle files or run"
72 72 @echo " sphinx-web build/pickle"
73 73 @echo "to start the sphinx-web server."
74 74
75 75 web: pickle
76 76
77 77 htmlhelp:
78 78 mkdir -p build/htmlhelp build/doctrees
79 79 $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) build/htmlhelp
80 80 @echo
81 81 @echo "Build finished; now you can run HTML Help Workshop with the" \
82 82 ".hhp project file in build/htmlhelp."
83 83
84 84 qthelp:
85 85 mkdir -p build/qthelp
86 86 $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) build/qthelp
87 87 @echo
88 88 @echo "Build finished; now you can run "qcollectiongenerator" with the" \
89 89 ".qhcp project file in $(BUILDDIR)/qthelp, like this:"
90 90 @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/IPython.qhcp"
91 91 @echo "To view the help file:"
92 92 @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/IPython.qhc"
93 93
94 94 latex: api
95 95 mkdir -p build/latex build/doctrees
96 96 $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) build/latex
97 97 @echo
98 98 @echo "Build finished; the LaTeX files are in build/latex."
99 99 @echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \
100 100 "run these through (pdf)latex."
101 101
102 102 changes:
103 103 mkdir -p build/changes build/doctrees
104 104 $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) build/changes
105 105 @echo
106 106 @echo "The overview file is in build/changes."
107 107
108 108 linkcheck:
109 109 mkdir -p build/linkcheck build/doctrees
110 110 $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) build/linkcheck
111 111 @echo
112 112 @echo "Link check complete; look for any errors in the above output " \
113 "or in build/linkcheck/output.txt."
113 "or in build/linkcheck/output.rst."
114 114
115 115 gitwash-update:
116 116 python ../tools/gitwash_dumper.py source/development ipython
117 cd source/development/gitwash && rename 's/.rst/.txt/' *.rst
118 117
119 118 nightly: dist
120 119 rsync -avH --delete dist/ ipython:www/doc/nightly
121 120
122 121 gh-pages: clean html
123 122 python gh-pages.py
124 123
125 124 texinfo:
126 125 mkdir -p $(BUILDDIR)/texinfo
127 126 $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
128 127 @echo
129 128 @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo."
130 129 @echo "Run \`make' in that directory to run these through makeinfo" \
131 130 "(use \`make info' here to do that automatically)."
132 131
133 132 info:
134 133 mkdir -p $(BUILDDIR)/texinfo
135 134 $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
136 135 @echo "Running Texinfo files through makeinfo..."
137 136 make -C $(BUILDDIR)/texinfo info
138 137 @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo."
1 NO CONTENT: file renamed from docs/README.txt to docs/README.rst
@@ -1,66 +1,66 b''
1 1 #!/usr/bin/env python
2 2 """Script to auto-generate our API docs.
3 3 """
4 4 # stdlib imports
5 5 import os
6 6 import sys
7 7
8 8 # local imports
9 9 sys.path.append(os.path.abspath('sphinxext'))
10 10 from apigen import ApiDocWriter
11 11
12 12 #*****************************************************************************
13 13 if __name__ == '__main__':
14 14 pjoin = os.path.join
15 15 package = 'IPython'
16 16 outdir = pjoin('source','api','generated')
17 docwriter = ApiDocWriter(package,rst_extension='.txt')
17 docwriter = ApiDocWriter(package,rst_extension='.rst')
18 18 # You have to escape the . here because . is a special char for regexps.
19 19 # You must do make clean if you change this!
20 20 docwriter.package_skip_patterns += [r'\.fixes$',
21 21 r'\.external$',
22 22 r'\.extensions',
23 23 r'\.kernel\.config',
24 24 r'\.attic',
25 25 r'\.quarantine',
26 26 r'\.deathrow',
27 27 r'\.config\.default',
28 28 r'\.config\.profile',
29 29 r'\.frontend',
30 30 r'\.gui',
31 31 r'\.kernel',
32 32 # For now, the zmq code has
33 33 # unconditional top-level code so it's
34 34 # not import safe. This needs fixing
35 35 r'\.zmq',
36 36 ]
37 37
38 38 docwriter.module_skip_patterns += [ r'\.core\.fakemodule',
39 39 r'\.testing\.iptest',
40 40 # Keeping these disabled is OK
41 41 r'\.parallel\.controller\.mongodb',
42 42 r'\.lib\.inputhookwx',
43 43 r'\.lib\.inputhookgtk',
44 44 r'\.cocoa',
45 45 r'\.ipdoctest',
46 46 r'\.Gnuplot',
47 47 r'\.frontend\.process\.winprocess',
48 48 r'\.frontend',
49 49 r'\.Shell',
50 50 ]
51 51
52 52 # If we don't have pexpect, we can't load irunner, so skip any code that
53 53 # depends on it
54 54 try:
55 55 import pexpect
56 56 except ImportError:
57 57 docwriter.module_skip_patterns += [r'\.lib\.irunner',
58 58 r'\.testing\.mkdoctests']
59 59 # Now, generate the outputs
60 60 docwriter.write_api_docs(outdir)
61 # Write index with .rst extension - we can include it, but Sphinx won't try
61 # Write index with .txt extension - we can include it, but Sphinx won't try
62 62 # to compile it
63 docwriter.write_index(outdir, 'gen.rst',
63 docwriter.write_index(outdir, 'gen.txt',
64 64 relative_to = pjoin('source','api')
65 65 )
66 66 print ('%d files written' % len(docwriter.written_modules))
@@ -1,71 +1,71 b''
1 1 @ECHO OFF
2 2 REM ~ Windows command line make file for Sphinx documentation
3 3
4 4 SETLOCAL
5 5
6 6 SET SPHINXOPTS=
7 7 SET SPHINXBUILD=sphinx-build
8 8 SET PAPER=
9 9 SET SRCDIR=source
10 10
11 11 IF "%PAPER%" == "" SET PAPER=a4
12 12 SET ALLSPHINXOPTS=-d build\doctrees -D latex_paper_size=%PAPER% %SPHINXOPTS% %SRCDIR%
13 13
14 14 FOR %%X IN (%SPHINXBUILD%.exe) DO SET P=%%~$PATH:X
15 15
16 16 FOR %%L IN (html pickle htmlhelp latex changes linkcheck) DO (
17 17 IF "%1" == "%%L" (
18 18 IF "%P%" == "" (
19 19 ECHO.
20 20 ECHO Error: Sphinx is not available. Please make sure it is correctly installed.
21 21 GOTO END
22 22 )
23 23 MD build\doctrees 2>NUL
24 24 MD build\%1 || GOTO DIR_EXIST
25 25 %SPHINXBUILD% -b %1 %ALLSPHINXOPTS% build\%1
26 26 IF NOT ERRORLEVEL 0 GOTO ERROR
27 27 ECHO.
28 28 ECHO Build finished. Results are in build\%1.
29 29 IF "%1" == "pickle" (
30 30 ECHO Now you can process the pickle files or run
31 31 ECHO sphinx-web build\pickle to start the sphinx-web server.
32 32 )
33 33 IF "%1" == "htmlhelp" (
34 34 ECHO Now you can run HTML Help Workshop with the
35 35 ECHO .hhp project file in build/htmlhelp.
36 36 )
37 37 IF "%1" == "linkcheck" (
38 38 ECHO Look for any errors in the above output
39 ECHO or in build\linkcheck\output.txt.
39 ECHO or in build\linkcheck\output.rst.
40 40 )
41 41 GOTO END
42 42 )
43 43 )
44 44
45 45 IF "%1" == "clean" (
46 46 RD /s /q build dist %SRCDIR%\api\generated 2>NUL
47 47 IF ERRORLEVEL 0 ECHO Build environment cleaned!
48 48 GOTO END
49 49 )
50 50
51 51 ECHO.
52 52 ECHO Please use "make [target]" where [target] is one of:
53 53 ECHO.
54 54 ECHO html to make standalone HTML files
55 55 ECHO pickle to make pickle files (usable by e.g. sphinx-web)
56 56 ECHO htmlhelp to make HTML files and a HTML help project
57 57 ECHO latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter
58 58 ECHO changes to make an overview over all changed/added/deprecated items
59 59 ECHO linkcheck to check all external links for integrity
60 60 GOTO END
61 61
62 62 :DIR_EXIST
63 63 ECHO.
64 64 ECHO Info: Run "make clean" to clean build environment
65 65
66 66 :ERROR
67 67 ECHO.
68 68 ECHO Error: Build process failed!
69 69
70 70 :END
71 71 ENDLOCAL No newline at end of file
1 NO CONTENT: file renamed from docs/source/about/credits.txt to docs/source/about/credits.rst
1 NO CONTENT: file renamed from docs/source/about/history.txt to docs/source/about/history.rst
1 NO CONTENT: file renamed from docs/source/about/index.txt to docs/source/about/index.rst
1 NO CONTENT: file renamed from docs/source/about/license_and_copyright.txt to docs/source/about/license_and_copyright.rst
@@ -1,12 +1,12 b''
1 1 .. _api-index:
2 2
3 3 ###################
4 4 The IPython API
5 5 ###################
6 6
7 7 .. htmlonly::
8 8
9 9 :Release: |version|
10 10 :Date: |today|
11 11
12 .. include:: generated/gen.rst
12 .. include:: generated/gen.txt
1 NO CONTENT: file renamed from docs/source/attic/parallel_task_old.txt to docs/source/attic/parallel_task_old.rst
@@ -1,228 +1,228 b''
1 1 # -*- coding: utf-8 -*-
2 2 #
3 3 # IPython documentation build configuration file.
4 4
5 5 # NOTE: This file has been edited manually from the auto-generated one from
6 6 # sphinx. Do NOT delete and re-generate. If any changes from sphinx are
7 7 # needed, generate a scratch one and merge by hand any new fields needed.
8 8
9 9 #
10 10 # This file is execfile()d with the current directory set to its containing dir.
11 11 #
12 12 # The contents of this file are pickled, so don't put values in the namespace
13 13 # that aren't pickleable (module imports are okay, they're removed automatically).
14 14 #
15 15 # All configuration values have a default value; values that are commented out
16 16 # serve to show the default value.
17 17
18 18 import sys, os
19 19
20 20 ON_RTD = os.environ.get('READTHEDOCS', None) == 'True'
21 21
22 22 if ON_RTD:
23 23 # Mock the presence of matplotlib, which we don't have on RTD
24 24 # see
25 25 # http://read-the-docs.readthedocs.org/en/latest/faq.html
26 26 tags.add('rtd')
27 27
28 28 # If your extensions are in another directory, add it here. If the directory
29 29 # is relative to the documentation root, use os.path.abspath to make it
30 30 # absolute, like shown here.
31 31 sys.path.insert(0, os.path.abspath('../sphinxext'))
32 32
33 33 # Import support for ipython console session syntax highlighting
34 34 # (lives IPython's sphinxext subpackage)
35 35 from IPython.sphinxext import ipython_console_highlighting
36 36
37 37 # We load the ipython release info into a dict by explicit execution
38 38 iprelease = {}
39 39 execfile('../../IPython/core/release.py',iprelease)
40 40
41 41 # General configuration
42 42 # ---------------------
43 43
44 44 # Add any Sphinx extension module names here, as strings. They can be extensions
45 45 # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
46 46 extensions = [
47 47 'matplotlib.sphinxext.mathmpl',
48 48 'matplotlib.sphinxext.only_directives',
49 49 'matplotlib.sphinxext.plot_directive',
50 50 'sphinx.ext.autodoc',
51 51 'sphinx.ext.doctest',
52 52 'sphinx.ext.inheritance_diagram',
53 53 'IPython.sphinxext.ipython_console_highlighting',
54 54 'IPython.sphinxext.ipython_directive',
55 55 'numpydoc', # to preprocess docstrings
56 56 'github', # for easy GitHub links
57 57 ]
58 58
59 59 if ON_RTD:
60 60 # Remove extensions not currently supported on RTD
61 61 extensions.remove('matplotlib.sphinxext.only_directives')
62 62 extensions.remove('matplotlib.sphinxext.mathmpl')
63 63 extensions.remove('matplotlib.sphinxext.plot_directive')
64 64 extensions.remove('IPython.sphinxext.ipython_directive')
65 65
66 66 # Add any paths that contain templates here, relative to this directory.
67 67 templates_path = ['_templates']
68 68
69 69 # The suffix of source filenames.
70 source_suffix = '.txt'
70 source_suffix = '.rst'
71 71
72 72 # The master toctree document.
73 73 master_doc = 'index'
74 74
75 75 # General substitutions.
76 76 project = 'IPython'
77 77 copyright = '2008, The IPython Development Team'
78 78
79 79 # ghissue config
80 80 github_project_url = "https://github.com/ipython/ipython"
81 81
82 82 # The default replacements for |version| and |release|, also used in various
83 83 # other places throughout the built documents.
84 84 #
85 85 # The full version, including alpha/beta/rc tags.
86 86 release = iprelease['version']
87 87 # The short X.Y version.
88 88 version = '.'.join(release.split('.',2)[:2])
89 89
90 90
91 91 # There are two options for replacing |today|: either, you set today to some
92 92 # non-false value, then it is used:
93 93 #today = ''
94 94 # Else, today_fmt is used as the format for a strftime call.
95 95 today_fmt = '%B %d, %Y'
96 96
97 97 # List of documents that shouldn't be included in the build.
98 98 #unused_docs = []
99 99
100 100 # List of directories, relative to source directories, that shouldn't be searched
101 101 # for source files.
102 102 exclude_dirs = ['attic']
103 103
104 104 # If true, '()' will be appended to :func: etc. cross-reference text.
105 105 #add_function_parentheses = True
106 106
107 107 # If true, the current module name will be prepended to all description
108 108 # unit titles (such as .. function::).
109 109 #add_module_names = True
110 110
111 111 # If true, sectionauthor and moduleauthor directives will be shown in the
112 112 # output. They are ignored by default.
113 113 #show_authors = False
114 114
115 115 # The name of the Pygments (syntax highlighting) style to use.
116 116 pygments_style = 'sphinx'
117 117
118 118
119 119 # Options for HTML output
120 120 # -----------------------
121 121
122 122 # The style sheet to use for HTML and HTML Help pages. A file of that name
123 123 # must exist either in Sphinx' static/ path, or in one of the custom paths
124 124 # given in html_static_path.
125 125 html_style = 'default.css'
126 126
127 127 # The name for this set of Sphinx documents. If None, it defaults to
128 128 # "<project> v<release> documentation".
129 129 #html_title = None
130 130
131 131 # The name of an image file (within the static path) to place at the top of
132 132 # the sidebar.
133 133 #html_logo = None
134 134
135 135 # Add any paths that contain custom static files (such as style sheets) here,
136 136 # relative to this directory. They are copied after the builtin static files,
137 137 # so a file named "default.css" will overwrite the builtin "default.css".
138 138 html_static_path = ['_static']
139 139
140 140 # If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
141 141 # using the given strftime format.
142 142 html_last_updated_fmt = '%b %d, %Y'
143 143
144 144 # If true, SmartyPants will be used to convert quotes and dashes to
145 145 # typographically correct entities.
146 146 #html_use_smartypants = True
147 147
148 148 # Custom sidebar templates, maps document names to template names.
149 149 #html_sidebars = {}
150 150
151 151 # Additional templates that should be rendered to pages, maps page names to
152 152 # template names.
153 153 #html_additional_pages = {}
154 154
155 155 # If false, no module index is generated.
156 156 #html_use_modindex = True
157 157
158 158 # If true, the reST sources are included in the HTML build as _sources/<name>.
159 159 #html_copy_source = True
160 160
161 161 # If true, an OpenSearch description file will be output, and all pages will
162 162 # contain a <link> tag referring to it. The value of this option must be the
163 163 # base URL from which the finished HTML is served.
164 164 #html_use_opensearch = ''
165 165
166 166 # If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml").
167 167 #html_file_suffix = ''
168 168
169 169 # Output file base name for HTML help builder.
170 170 htmlhelp_basename = 'ipythondoc'
171 171
172 172
173 173 # Options for LaTeX output
174 174 # ------------------------
175 175
176 176 # The paper size ('letter' or 'a4').
177 177 latex_paper_size = 'letter'
178 178
179 179 # The font size ('10pt', '11pt' or '12pt').
180 180 latex_font_size = '11pt'
181 181
182 182 # Grouping the document tree into LaTeX files. List of tuples
183 183 # (source start file, target name, title, author, document class [howto/manual]).
184 184
185 185 latex_documents = [
186 186 ('index', 'ipython.tex', 'IPython Documentation',
187 187 ur"""The IPython Development Team""", 'manual', True),
188 188 ('parallel/winhpc_index', 'winhpc_whitepaper.tex',
189 189 'Using IPython on Windows HPC Server 2008',
190 190 ur"Brian E. Granger", 'manual', True)
191 191 ]
192 192
193 193 # The name of an image file (relative to this directory) to place at the top of
194 194 # the title page.
195 195 #latex_logo = None
196 196
197 197 # For "manual" documents, if this is true, then toplevel headings are parts,
198 198 # not chapters.
199 199 #latex_use_parts = False
200 200
201 201 # Additional stuff for the LaTeX preamble.
202 202 #latex_preamble = ''
203 203
204 204 # Documents to append as an appendix to all manuals.
205 205 #latex_appendices = []
206 206
207 207 # If false, no module index is generated.
208 208 latex_use_modindex = True
209 209
210 210
211 211 # Options for texinfo output
212 212 # --------------------------
213 213
214 214 texinfo_documents = [
215 215 (master_doc, 'ipython', 'IPython Documentation',
216 216 'The IPython Development Team',
217 217 'IPython',
218 218 'IPython Documentation',
219 219 'Programming',
220 220 1),
221 221 ]
222 222
223 223
224 224 # Cleanup
225 225 # -------
226 226 # delete release info to avoid pickling errors from sphinx
227 227
228 228 del iprelease
1 NO CONTENT: file renamed from docs/source/config/editors.txt to docs/source/config/editors.rst
1 NO CONTENT: file renamed from docs/source/config/extensions/autoreload.txt to docs/source/config/extensions/autoreload.rst
1 NO CONTENT: file renamed from docs/source/config/extensions/cythonmagic.txt to docs/source/config/extensions/cythonmagic.rst
1 NO CONTENT: file renamed from docs/source/config/extensions/index.txt to docs/source/config/extensions/index.rst
1 NO CONTENT: file renamed from docs/source/config/extensions/octavemagic.txt to docs/source/config/extensions/octavemagic.rst
1 NO CONTENT: file renamed from docs/source/config/extensions/rmagic.txt to docs/source/config/extensions/rmagic.rst
1 NO CONTENT: file renamed from docs/source/config/extensions/storemagic.txt to docs/source/config/extensions/storemagic.rst
1 NO CONTENT: file renamed from docs/source/config/extensions/sympyprinting.txt to docs/source/config/extensions/sympyprinting.rst
1 NO CONTENT: file renamed from docs/source/config/inputtransforms.txt to docs/source/config/inputtransforms.rst
1 NO CONTENT: file renamed from docs/source/config/integrating.txt to docs/source/config/integrating.rst
1 NO CONTENT: file renamed from docs/source/config/ipython.txt to docs/source/config/ipython.rst
1 NO CONTENT: file renamed from docs/source/config/old.txt to docs/source/config/old.rst
1 NO CONTENT: file renamed from docs/source/config/overview.txt to docs/source/config/overview.rst
1 NO CONTENT: file renamed from docs/source/development/gitwash/configure_git.txt to docs/source/development/gitwash/configure_git.rst
1 NO CONTENT: file renamed from docs/source/development/gitwash/development_workflow.txt to docs/source/development/gitwash/development_workflow.rst
1 NO CONTENT: file renamed from docs/source/development/gitwash/dot2_dot3.txt to docs/source/development/gitwash/dot2_dot3.rst
1 NO CONTENT: file renamed from docs/source/development/gitwash/following_latest.txt to docs/source/development/gitwash/following_latest.rst
1 NO CONTENT: file renamed from docs/source/development/gitwash/forking_hell.txt to docs/source/development/gitwash/forking_hell.rst
1 NO CONTENT: file renamed from docs/source/development/gitwash/git_development.txt to docs/source/development/gitwash/git_development.rst
1 NO CONTENT: file renamed from docs/source/development/gitwash/git_install.txt to docs/source/development/gitwash/git_install.rst
1 NO CONTENT: file renamed from docs/source/development/gitwash/git_resources.txt to docs/source/development/gitwash/git_resources.rst
1 NO CONTENT: file renamed from docs/source/development/gitwash/index.txt to docs/source/development/gitwash/index.rst
1 NO CONTENT: file renamed from docs/source/development/gitwash/patching.txt to docs/source/development/gitwash/patching.rst
1 NO CONTENT: file renamed from docs/source/development/gitwash/set_up_fork.txt to docs/source/development/gitwash/set_up_fork.rst
@@ -1,26 +1,26 b''
1 1 .. _developer_guide:
2 2
3 3 =========================
4 4 IPython developer's guide
5 5 =========================
6 6
7 7 This are two categories of developer focused documentation:
8 8
9 9 1. Documentation for developers of *IPython itself*.
10 10 2. Documentation for developers of third party tools and libraries
11 11 that use IPython.
12 12
13 13 This part of our documentation only contains information in the second category.
14 14
15 15 Developers interested in working on IPython itself should consult
16 16 our `developer information <https://github.com/ipython/ipython/wiki/Dev:-Index>`_
17 17 on the IPython GitHub wiki.
18 18
19 19 .. toctree::
20 20 :maxdepth: 1
21 21
22 22
23 gitwash/index.txt
24 messaging.txt
25 parallel_messages.txt
26 parallel_connections.txt
23 gitwash/index
24 messaging
25 parallel_messages
26 parallel_connections
1 NO CONTENT: file renamed from docs/source/development/inputhook_app.txt to docs/source/development/inputhook_app.rst
@@ -1,1061 +1,1061 b''
1 1 .. _messaging:
2 2
3 3 ======================
4 4 Messaging in IPython
5 5 ======================
6 6
7 7
8 8 Introduction
9 9 ============
10 10
11 11 This document explains the basic communications design and messaging
12 12 specification for how the various IPython objects interact over a network
13 13 transport. The current implementation uses the ZeroMQ_ library for messaging
14 14 within and between hosts.
15 15
16 16 .. Note::
17 17
18 18 This document should be considered the authoritative description of the
19 19 IPython messaging protocol, and all developers are strongly encouraged to
20 20 keep it updated as the implementation evolves, so that we have a single
21 21 common reference for all protocol details.
22 22
23 23 The basic design is explained in the following diagram:
24 24
25 25 .. image:: figs/frontend-kernel.png
26 26 :width: 450px
27 27 :alt: IPython kernel/frontend messaging architecture.
28 28 :align: center
29 29 :target: ../_images/frontend-kernel.png
30 30
31 31 A single kernel can be simultaneously connected to one or more frontends. The
32 32 kernel has three sockets that serve the following functions:
33 33
34 34 1. stdin: this ROUTER socket is connected to all frontends, and it allows
35 35 the kernel to request input from the active frontend when :func:`raw_input` is called.
36 36 The frontend that executed the code has a DEALER socket that acts as a 'virtual keyboard'
37 37 for the kernel while this communication is happening (illustrated in the
38 38 figure by the black outline around the central keyboard). In practice,
39 39 frontends may display such kernel requests using a special input widget or
40 40 otherwise indicating that the user is to type input for the kernel instead
41 41 of normal commands in the frontend.
42 42
43 43 2. Shell: this single ROUTER socket allows multiple incoming connections from
44 44 frontends, and this is the socket where requests for code execution, object
45 45 information, prompts, etc. are made to the kernel by any frontend. The
46 46 communication on this socket is a sequence of request/reply actions from
47 47 each frontend and the kernel.
48 48
49 49 3. IOPub: this socket is the 'broadcast channel' where the kernel publishes all
50 50 side effects (stdout, stderr, etc.) as well as the requests coming from any
51 51 client over the shell socket and its own requests on the stdin socket. There
52 52 are a number of actions in Python which generate side effects: :func:`print`
53 53 writes to ``sys.stdout``, errors generate tracebacks, etc. Additionally, in
54 54 a multi-client scenario, we want all frontends to be able to know what each
55 55 other has sent to the kernel (this can be useful in collaborative scenarios,
56 56 for example). This socket allows both side effects and the information
57 57 about communications taking place with one client over the shell channel
58 58 to be made available to all clients in a uniform manner.
59 59
60 60 All messages are tagged with enough information (details below) for clients
61 61 to know which messages come from their own interaction with the kernel and
62 62 which ones are from other clients, so they can display each type
63 63 appropriately.
64 64
65 65 The actual format of the messages allowed on each of these channels is
66 66 specified below. Messages are dicts of dicts with string keys and values that
67 67 are reasonably representable in JSON. Our current implementation uses JSON
68 68 explicitly as its message format, but this shouldn't be considered a permanent
69 69 feature. As we've discovered that JSON has non-trivial performance issues due
70 70 to excessive copying, we may in the future move to a pure pickle-based raw
71 71 message format. However, it should be possible to easily convert from the raw
72 72 objects to JSON, since we may have non-python clients (e.g. a web frontend).
73 73 As long as it's easy to make a JSON version of the objects that is a faithful
74 74 representation of all the data, we can communicate with such clients.
75 75
76 76 .. Note::
77 77
78 78 Not all of these have yet been fully fleshed out, but the key ones are, see
79 79 kernel and frontend files for actual implementation details.
80 80
81 81 General Message Format
82 82 ======================
83 83
84 84 A message is defined by the following four-dictionary structure::
85 85
86 86 {
87 87 # The message header contains a pair of unique identifiers for the
88 88 # originating session and the actual message id, in addition to the
89 89 # username for the process that generated the message. This is useful in
90 90 # collaborative settings where multiple users may be interacting with the
91 91 # same kernel simultaneously, so that frontends can label the various
92 92 # messages in a meaningful way.
93 93 'header' : {
94 94 'msg_id' : uuid,
95 95 'username' : str,
96 96 'session' : uuid,
97 97 # All recognized message type strings are listed below.
98 98 'msg_type' : str,
99 99 },
100 100
101 101 # In a chain of messages, the header from the parent is copied so that
102 102 # clients can track where messages come from.
103 103 'parent_header' : dict,
104 104
105 105 # Any metadata associated with the message.
106 106 'metadata' : dict,
107 107
108 108 # The actual content of the message must be a dict, whose structure
109 109 # depends on the message type.
110 110 'content' : dict,
111 111 }
112 112
113 113 The Wire Protocol
114 114 =================
115 115
116 116
117 117 This message format exists at a high level,
118 118 but does not describe the actual *implementation* at the wire level in zeromq.
119 119 The canonical implementation of the message spec is our :class:`~IPython.kernel.zmq.session.Session` class.
120 120
121 121 .. note::
122 122
123 123 This section should only be relevant to non-Python consumers of the protocol.
124 124 Python consumers should simply import and use IPython's own implementation of the wire protocol
125 125 in the :class:`IPython.kernel.zmq.session.Session` object.
126 126
127 127 Every message is serialized to a sequence of at least six blobs of bytes:
128 128
129 129 .. sourcecode:: python
130 130
131 131 [
132 132 b'u-u-i-d', # zmq identity(ies)
133 133 b'<IDS|MSG>', # delimiter
134 134 b'baddad42', # HMAC signature
135 135 b'{header}', # serialized header dict
136 136 b'{parent_header}', # serialized parent header dict
137 137 b'{metadata}', # serialized metadata dict
138 138 b'{content}, # serialized content dict
139 139 b'blob', # extra raw data buffer(s)
140 140 ...
141 141 ]
142 142
143 143 The front of the message is the ZeroMQ routing prefix,
144 144 which can be zero or more socket identities.
145 145 This is every piece of the message prior to the delimiter key ``<IDS|MSG>``.
146 146 In the case of IOPub, there should be just one prefix component,
147 147 which is the topic for IOPub subscribers, e.g. ``pyout``, ``display_data``.
148 148
149 149 .. note::
150 150
151 151 In most cases, the IOPub topics are irrelevant and completely ignored,
152 152 because frontends just subscribe to all topics.
153 153 The convention used in the IPython kernel is to use the msg_type as the topic,
154 154 and possibly extra information about the message, e.g. ``pyout`` or ``stream.stdout``
155 155
156 156 After the delimiter is the `HMAC`_ signature of the message, used for authentication.
157 157 If authentication is disabled, this should be an empty string.
158 158 By default, the hashing function used for computing these signatures is sha256.
159 159
160 160 .. _HMAC: http://en.wikipedia.org/wiki/HMAC
161 161
162 162 .. note::
163 163
164 164 To disable authentication and signature checking,
165 165 set the `key` field of a connection file to an empty string.
166 166
167 167 The signature is the HMAC hex digest of the concatenation of:
168 168
169 169 - A shared key (typically the ``key`` field of a connection file)
170 170 - The serialized header dict
171 171 - The serialized parent header dict
172 172 - The serialized metadata dict
173 173 - The serialized content dict
174 174
175 175 In Python, this is implemented via:
176 176
177 177 .. sourcecode:: python
178 178
179 179 # once:
180 180 digester = HMAC(key, digestmod=hashlib.sha256)
181 181
182 182 # for each message
183 183 d = digester.copy()
184 184 for serialized_dict in (header, parent, metadata, content):
185 185 d.update(serialized_dict)
186 186 signature = d.hexdigest()
187 187
188 188 After the signature is the actual message, always in four frames of bytes.
189 189 The four dictionaries that compose a message are serialized separately,
190 190 in the order of header, parent header, metadata, and content.
191 191 These can be serialized by any function that turns a dict into bytes.
192 192 The default and most common serialization is JSON, but msgpack and pickle
193 193 are common alternatives.
194 194
195 195 After the serialized dicts are zero to many raw data buffers,
196 196 which can be used by message types that support binary data (mainly apply and data_pub).
197 197
198 198
199 199 Python functional API
200 200 =====================
201 201
202 202 As messages are dicts, they map naturally to a ``func(**kw)`` call form. We
203 203 should develop, at a few key points, functional forms of all the requests that
204 204 take arguments in this manner and automatically construct the necessary dict
205 205 for sending.
206 206
207 207 In addition, the Python implementation of the message specification extends
208 208 messages upon deserialization to the following form for convenience::
209 209
210 210 {
211 211 'header' : dict,
212 212 # The msg's unique identifier and type are always stored in the header,
213 213 # but the Python implementation copies them to the top level.
214 214 'msg_id' : uuid,
215 215 'msg_type' : str,
216 216 'parent_header' : dict,
217 217 'content' : dict,
218 218 'metadata' : dict,
219 219 }
220 220
221 221 All messages sent to or received by any IPython process should have this
222 222 extended structure.
223 223
224 224
225 225 Messages on the shell ROUTER/DEALER sockets
226 226 ===========================================
227 227
228 228 .. _execute:
229 229
230 230 Execute
231 231 -------
232 232
233 233 This message type is used by frontends to ask the kernel to execute code on
234 234 behalf of the user, in a namespace reserved to the user's variables (and thus
235 235 separate from the kernel's own internal code and variables).
236 236
237 237 Message type: ``execute_request``::
238 238
239 239 content = {
240 240 # Source code to be executed by the kernel, one or more lines.
241 241 'code' : str,
242 242
243 243 # A boolean flag which, if True, signals the kernel to execute
244 244 # this code as quietly as possible. This means that the kernel
245 245 # will compile the code with 'exec' instead of 'single' (so
246 246 # sys.displayhook will not fire), forces store_history to be False,
247 247 # and will *not*:
248 248 # - broadcast exceptions on the PUB socket
249 249 # - do any logging
250 250 #
251 251 # The default is False.
252 252 'silent' : bool,
253 253
254 254 # A boolean flag which, if True, signals the kernel to populate history
255 255 # The default is True if silent is False. If silent is True, store_history
256 256 # is forced to be False.
257 257 'store_history' : bool,
258 258
259 259 # A list of variable names from the user's namespace to be retrieved.
260 260 # What returns is a rich representation of each variable (dict keyed by name).
261 261 # See the display_data content for the structure of the representation data.
262 262 'user_variables' : list,
263 263
264 264 # Similarly, a dict mapping names to expressions to be evaluated in the
265 265 # user's dict.
266 266 'user_expressions' : dict,
267 267
268 268 # Some frontends (e.g. the Notebook) do not support stdin requests. If
269 269 # raw_input is called from code executed from such a frontend, a
270 270 # StdinNotImplementedError will be raised.
271 271 'allow_stdin' : True,
272 272
273 273 }
274 274
275 275 The ``code`` field contains a single string (possibly multiline). The kernel
276 276 is responsible for splitting this into one or more independent execution blocks
277 277 and deciding whether to compile these in 'single' or 'exec' mode (see below for
278 278 detailed execution semantics).
279 279
280 280 The ``user_`` fields deserve a detailed explanation. In the past, IPython had
281 281 the notion of a prompt string that allowed arbitrary code to be evaluated, and
282 282 this was put to good use by many in creating prompts that displayed system
283 283 status, path information, and even more esoteric uses like remote instrument
284 284 status acquired over the network. But now that IPython has a clean separation
285 285 between the kernel and the clients, the kernel has no prompt knowledge; prompts
286 286 are a frontend-side feature, and it should be even possible for different
287 287 frontends to display different prompts while interacting with the same kernel.
288 288
289 289 The kernel now provides the ability to retrieve data from the user's namespace
290 290 after the execution of the main ``code``, thanks to two fields in the
291 291 ``execute_request`` message:
292 292
293 293 - ``user_variables``: If only variables from the user's namespace are needed, a
294 294 list of variable names can be passed and a dict with these names as keys and
295 295 their :func:`repr()` as values will be returned.
296 296
297 297 - ``user_expressions``: For more complex expressions that require function
298 298 evaluations, a dict can be provided with string keys and arbitrary python
299 299 expressions as values. The return message will contain also a dict with the
300 300 same keys and the :func:`repr()` of the evaluated expressions as value.
301 301
302 302 With this information, frontends can display any status information they wish
303 303 in the form that best suits each frontend (a status line, a popup, inline for a
304 304 terminal, etc).
305 305
306 306 .. Note::
307 307
308 308 In order to obtain the current execution counter for the purposes of
309 309 displaying input prompts, frontends simply make an execution request with an
310 310 empty code string and ``silent=True``.
311 311
312 312 Execution semantics
313 313 ~~~~~~~~~~~~~~~~~~~
314 314
315 315 When the silent flag is false, the execution of use code consists of the
316 316 following phases (in silent mode, only the ``code`` field is executed):
317 317
318 318 1. Run the ``pre_runcode_hook``.
319 319
320 320 2. Execute the ``code`` field, see below for details.
321 321
322 322 3. If #2 succeeds, compute ``user_variables`` and ``user_expressions`` are
323 323 computed. This ensures that any error in the latter don't harm the main
324 324 code execution.
325 325
326 326 4. Call any method registered with :meth:`register_post_execute`.
327 327
328 328 .. warning::
329 329
330 330 The API for running code before/after the main code block is likely to
331 331 change soon. Both the ``pre_runcode_hook`` and the
332 332 :meth:`register_post_execute` are susceptible to modification, as we find a
333 333 consistent model for both.
334 334
335 335 To understand how the ``code`` field is executed, one must know that Python
336 336 code can be compiled in one of three modes (controlled by the ``mode`` argument
337 337 to the :func:`compile` builtin):
338 338
339 339 *single*
340 340 Valid for a single interactive statement (though the source can contain
341 341 multiple lines, such as a for loop). When compiled in this mode, the
342 342 generated bytecode contains special instructions that trigger the calling of
343 343 :func:`sys.displayhook` for any expression in the block that returns a value.
344 344 This means that a single statement can actually produce multiple calls to
345 345 :func:`sys.displayhook`, if for example it contains a loop where each
346 346 iteration computes an unassigned expression would generate 10 calls::
347 347
348 348 for i in range(10):
349 349 i**2
350 350
351 351 *exec*
352 352 An arbitrary amount of source code, this is how modules are compiled.
353 353 :func:`sys.displayhook` is *never* implicitly called.
354 354
355 355 *eval*
356 356 A single expression that returns a value. :func:`sys.displayhook` is *never*
357 357 implicitly called.
358 358
359 359
360 360 The ``code`` field is split into individual blocks each of which is valid for
361 361 execution in 'single' mode, and then:
362 362
363 363 - If there is only a single block: it is executed in 'single' mode.
364 364
365 365 - If there is more than one block:
366 366
367 367 * if the last one is a single line long, run all but the last in 'exec' mode
368 368 and the very last one in 'single' mode. This makes it easy to type simple
369 369 expressions at the end to see computed values.
370 370
371 371 * if the last one is no more than two lines long, run all but the last in
372 372 'exec' mode and the very last one in 'single' mode. This makes it easy to
373 373 type simple expressions at the end to see computed values. - otherwise
374 374 (last one is also multiline), run all in 'exec' mode
375 375
376 376 * otherwise (last one is also multiline), run all in 'exec' mode as a single
377 377 unit.
378 378
379 379 Any error in retrieving the ``user_variables`` or evaluating the
380 380 ``user_expressions`` will result in a simple error message in the return fields
381 381 of the form::
382 382
383 383 [ERROR] ExceptionType: Exception message
384 384
385 385 The user can simply send the same variable name or expression for evaluation to
386 386 see a regular traceback.
387 387
388 388 Errors in any registered post_execute functions are also reported similarly,
389 389 and the failing function is removed from the post_execution set so that it does
390 390 not continue triggering failures.
391 391
392 392 Upon completion of the execution request, the kernel *always* sends a reply,
393 393 with a status code indicating what happened and additional data depending on
394 394 the outcome. See :ref:`below <execution_results>` for the possible return
395 395 codes and associated data.
396 396
397 397
398 398 Execution counter (old prompt number)
399 399 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
400 400
401 401 The kernel has a single, monotonically increasing counter of all execution
402 402 requests that are made with ``store_history=True``. This counter is used to populate
403 403 the ``In[n]``, ``Out[n]`` and ``_n`` variables, so clients will likely want to
404 404 display it in some form to the user, which will typically (but not necessarily)
405 405 be done in the prompts. The value of this counter will be returned as the
406 406 ``execution_count`` field of all ``execute_reply`` messages.
407 407
408 408 .. _execution_results:
409 409
410 410 Execution results
411 411 ~~~~~~~~~~~~~~~~~
412 412
413 413 Message type: ``execute_reply``::
414 414
415 415 content = {
416 416 # One of: 'ok' OR 'error' OR 'abort'
417 417 'status' : str,
418 418
419 419 # The global kernel counter that increases by one with each request that
420 420 # stores history. This will typically be used by clients to display
421 421 # prompt numbers to the user. If the request did not store history, this will
422 422 # be the current value of the counter in the kernel.
423 423 'execution_count' : int,
424 424 }
425 425
426 426 When status is 'ok', the following extra fields are present::
427 427
428 428 {
429 429 # 'payload' will be a list of payload dicts.
430 430 # Each execution payload is a dict with string keys that may have been
431 431 # produced by the code being executed. It is retrieved by the kernel at
432 432 # the end of the execution and sent back to the front end, which can take
433 433 # action on it as needed. See main text for further details.
434 434 'payload' : list(dict),
435 435
436 436 # Results for the user_variables and user_expressions.
437 437 'user_variables' : dict,
438 438 'user_expressions' : dict,
439 439 }
440 440
441 441 .. admonition:: Execution payloads
442 442
443 443 The notion of an 'execution payload' is different from a return value of a
444 444 given set of code, which normally is just displayed on the pyout stream
445 445 through the PUB socket. The idea of a payload is to allow special types of
446 446 code, typically magics, to populate a data container in the IPython kernel
447 447 that will be shipped back to the caller via this channel. The kernel
448 448 has an API for this in the PayloadManager::
449 449
450 450 ip.payload_manager.write_payload(payload_dict)
451 451
452 452 which appends a dictionary to the list of payloads.
453 453
454 454
455 455 When status is 'error', the following extra fields are present::
456 456
457 457 {
458 458 'ename' : str, # Exception name, as a string
459 459 'evalue' : str, # Exception value, as a string
460 460
461 461 # The traceback will contain a list of frames, represented each as a
462 462 # string. For now we'll stick to the existing design of ultraTB, which
463 463 # controls exception level of detail statefully. But eventually we'll
464 464 # want to grow into a model where more information is collected and
465 465 # packed into the traceback object, with clients deciding how little or
466 466 # how much of it to unpack. But for now, let's start with a simple list
467 467 # of strings, since that requires only minimal changes to ultratb as
468 468 # written.
469 469 'traceback' : list,
470 470 }
471 471
472 472
473 473 When status is 'abort', there are for now no additional data fields. This
474 474 happens when the kernel was interrupted by a signal.
475 475
476 476
477 477 Object information
478 478 ------------------
479 479
480 480 One of IPython's most used capabilities is the introspection of Python objects
481 481 in the user's namespace, typically invoked via the ``?`` and ``??`` characters
482 482 (which in reality are shorthands for the ``%pinfo`` magic). This is used often
483 483 enough that it warrants an explicit message type, especially because frontends
484 484 may want to get object information in response to user keystrokes (like Tab or
485 485 F1) besides from the user explicitly typing code like ``x??``.
486 486
487 487 Message type: ``object_info_request``::
488 488
489 489 content = {
490 490 # The (possibly dotted) name of the object to be searched in all
491 491 # relevant namespaces
492 492 'oname' : str,
493 493
494 494 # The level of detail desired. The default (0) is equivalent to typing
495 495 # 'x?' at the prompt, 1 is equivalent to 'x??'.
496 496 'detail_level' : int,
497 497 }
498 498
499 499 The returned information will be a dictionary with keys very similar to the
500 500 field names that IPython prints at the terminal.
501 501
502 502 Message type: ``object_info_reply``::
503 503
504 504 content = {
505 505 # The name the object was requested under
506 506 'name' : str,
507 507
508 508 # Boolean flag indicating whether the named object was found or not. If
509 509 # it's false, all other fields will be empty.
510 510 'found' : bool,
511 511
512 512 # Flags for magics and system aliases
513 513 'ismagic' : bool,
514 514 'isalias' : bool,
515 515
516 516 # The name of the namespace where the object was found ('builtin',
517 517 # 'magics', 'alias', 'interactive', etc.)
518 518 'namespace' : str,
519 519
520 520 # The type name will be type.__name__ for normal Python objects, but it
521 521 # can also be a string like 'Magic function' or 'System alias'
522 522 'type_name' : str,
523 523
524 524 # The string form of the object, possibly truncated for length if
525 525 # detail_level is 0
526 526 'string_form' : str,
527 527
528 528 # For objects with a __class__ attribute this will be set
529 529 'base_class' : str,
530 530
531 531 # For objects with a __len__ attribute this will be set
532 532 'length' : int,
533 533
534 534 # If the object is a function, class or method whose file we can find,
535 535 # we give its full path
536 536 'file' : str,
537 537
538 538 # For pure Python callable objects, we can reconstruct the object
539 539 # definition line which provides its call signature. For convenience this
540 540 # is returned as a single 'definition' field, but below the raw parts that
541 541 # compose it are also returned as the argspec field.
542 542 'definition' : str,
543 543
544 544 # The individual parts that together form the definition string. Clients
545 545 # with rich display capabilities may use this to provide a richer and more
546 546 # precise representation of the definition line (e.g. by highlighting
547 547 # arguments based on the user's cursor position). For non-callable
548 548 # objects, this field is empty.
549 549 'argspec' : { # The names of all the arguments
550 550 args : list,
551 551 # The name of the varargs (*args), if any
552 552 varargs : str,
553 553 # The name of the varkw (**kw), if any
554 554 varkw : str,
555 555 # The values (as strings) of all default arguments. Note
556 556 # that these must be matched *in reverse* with the 'args'
557 557 # list above, since the first positional args have no default
558 558 # value at all.
559 559 defaults : list,
560 560 },
561 561
562 562 # For instances, provide the constructor signature (the definition of
563 563 # the __init__ method):
564 564 'init_definition' : str,
565 565
566 566 # Docstrings: for any object (function, method, module, package) with a
567 567 # docstring, we show it. But in addition, we may provide additional
568 568 # docstrings. For example, for instances we will show the constructor
569 569 # and class docstrings as well, if available.
570 570 'docstring' : str,
571 571
572 572 # For instances, provide the constructor and class docstrings
573 573 'init_docstring' : str,
574 574 'class_docstring' : str,
575 575
576 576 # If it's a callable object whose call method has a separate docstring and
577 577 # definition line:
578 578 'call_def' : str,
579 579 'call_docstring' : str,
580 580
581 581 # If detail_level was 1, we also try to find the source code that
582 582 # defines the object, if possible. The string 'None' will indicate
583 583 # that no source was found.
584 584 'source' : str,
585 585 }
586 586
587 587
588 588 Complete
589 589 --------
590 590
591 591 Message type: ``complete_request``::
592 592
593 593 content = {
594 594 # The text to be completed, such as 'a.is'
595 595 # this may be an empty string if the frontend does not do any lexing,
596 596 # in which case the kernel must figure out the completion
597 597 # based on 'line' and 'cursor_pos'.
598 598 'text' : str,
599 599
600 600 # The full line, such as 'print a.is'. This allows completers to
601 601 # make decisions that may require information about more than just the
602 602 # current word.
603 603 'line' : str,
604 604
605 605 # The entire block of text where the line is. This may be useful in the
606 606 # case of multiline completions where more context may be needed. Note: if
607 607 # in practice this field proves unnecessary, remove it to lighten the
608 608 # messages.
609 609
610 610 'block' : str or null/None,
611 611
612 612 # The position of the cursor where the user hit 'TAB' on the line.
613 613 'cursor_pos' : int,
614 614 }
615 615
616 616 Message type: ``complete_reply``::
617 617
618 618 content = {
619 619 # The list of all matches to the completion request, such as
620 620 # ['a.isalnum', 'a.isalpha'] for the above example.
621 621 'matches' : list,
622 622
623 623 # the substring of the matched text
624 624 # this is typically the common prefix of the matches,
625 625 # and the text that is already in the block that would be replaced by the full completion.
626 626 # This would be 'a.is' in the above example.
627 627 'text' : str,
628 628
629 629 # status should be 'ok' unless an exception was raised during the request,
630 630 # in which case it should be 'error', along with the usual error message content
631 631 # in other messages.
632 632 'status' : 'ok'
633 633 }
634 634
635 635
636 636 History
637 637 -------
638 638
639 639 For clients to explicitly request history from a kernel. The kernel has all
640 640 the actual execution history stored in a single location, so clients can
641 641 request it from the kernel when needed.
642 642
643 643 Message type: ``history_request``::
644 644
645 645 content = {
646 646
647 647 # If True, also return output history in the resulting dict.
648 648 'output' : bool,
649 649
650 650 # If True, return the raw input history, else the transformed input.
651 651 'raw' : bool,
652 652
653 653 # So far, this can be 'range', 'tail' or 'search'.
654 654 'hist_access_type' : str,
655 655
656 656 # If hist_access_type is 'range', get a range of input cells. session can
657 657 # be a positive session number, or a negative number to count back from
658 658 # the current session.
659 659 'session' : int,
660 660 # start and stop are line numbers within that session.
661 661 'start' : int,
662 662 'stop' : int,
663 663
664 664 # If hist_access_type is 'tail' or 'search', get the last n cells.
665 665 'n' : int,
666 666
667 667 # If hist_access_type is 'search', get cells matching the specified glob
668 668 # pattern (with * and ? as wildcards).
669 669 'pattern' : str,
670 670
671 671 # If hist_access_type is 'search' and unique is true, do not
672 672 # include duplicated history. Default is false.
673 673 'unique' : bool,
674 674
675 675 }
676 676
677 677 .. versionadded:: 4.0
678 678 The key ``unique`` for ``history_request``.
679 679
680 680 Message type: ``history_reply``::
681 681
682 682 content = {
683 683 # A list of 3 tuples, either:
684 684 # (session, line_number, input) or
685 685 # (session, line_number, (input, output)),
686 686 # depending on whether output was False or True, respectively.
687 687 'history' : list,
688 688 }
689 689
690 690
691 691 Connect
692 692 -------
693 693
694 694 When a client connects to the request/reply socket of the kernel, it can issue
695 695 a connect request to get basic information about the kernel, such as the ports
696 696 the other ZeroMQ sockets are listening on. This allows clients to only have
697 697 to know about a single port (the shell channel) to connect to a kernel.
698 698
699 699 Message type: ``connect_request``::
700 700
701 701 content = {
702 702 }
703 703
704 704 Message type: ``connect_reply``::
705 705
706 706 content = {
707 707 'shell_port' : int, # The port the shell ROUTER socket is listening on.
708 708 'iopub_port' : int, # The port the PUB socket is listening on.
709 709 'stdin_port' : int, # The port the stdin ROUTER socket is listening on.
710 710 'hb_port' : int, # The port the heartbeat socket is listening on.
711 711 }
712 712
713 713
714 714 Kernel info
715 715 -----------
716 716
717 717 If a client needs to know information about the kernel, it can
718 718 make a request of the kernel's information.
719 719 This message can be used to fetch core information of the
720 720 kernel, including language (e.g., Python), language version number and
721 721 IPython version number, and the IPython message spec version number.
722 722
723 723 Message type: ``kernel_info_request``::
724 724
725 725 content = {
726 726 }
727 727
728 728 Message type: ``kernel_info_reply``::
729 729
730 730 content = {
731 731 # Version of messaging protocol (mandatory).
732 732 # The first integer indicates major version. It is incremented when
733 733 # there is any backward incompatible change.
734 734 # The second integer indicates minor version. It is incremented when
735 735 # there is any backward compatible change.
736 736 'protocol_version': [int, int],
737 737
738 738 # IPython version number (optional).
739 739 # Non-python kernel backend may not have this version number.
740 740 # The last component is an extra field, which may be 'dev' or
741 741 # 'rc1' in development version. It is an empty string for
742 742 # released version.
743 743 'ipython_version': [int, int, int, str],
744 744
745 745 # Language version number (mandatory).
746 746 # It is Python version number (e.g., [2, 7, 3]) for the kernel
747 747 # included in IPython.
748 748 'language_version': [int, ...],
749 749
750 750 # Programming language in which kernel is implemented (mandatory).
751 751 # Kernel included in IPython returns 'python'.
752 752 'language': str,
753 753 }
754 754
755 755
756 756 Kernel shutdown
757 757 ---------------
758 758
759 759 The clients can request the kernel to shut itself down; this is used in
760 760 multiple cases:
761 761
762 762 - when the user chooses to close the client application via a menu or window
763 763 control.
764 764 - when the user types 'exit' or 'quit' (or their uppercase magic equivalents).
765 765 - when the user chooses a GUI method (like the 'Ctrl-C' shortcut in the
766 766 IPythonQt client) to force a kernel restart to get a clean kernel without
767 767 losing client-side state like history or inlined figures.
768 768
769 769 The client sends a shutdown request to the kernel, and once it receives the
770 770 reply message (which is otherwise empty), it can assume that the kernel has
771 771 completed shutdown safely.
772 772
773 773 Upon their own shutdown, client applications will typically execute a last
774 774 minute sanity check and forcefully terminate any kernel that is still alive, to
775 775 avoid leaving stray processes in the user's machine.
776 776
777 777 Message type: ``shutdown_request``::
778 778
779 779 content = {
780 780 'restart' : bool # whether the shutdown is final, or precedes a restart
781 781 }
782 782
783 783 Message type: ``shutdown_reply``::
784 784
785 785 content = {
786 786 'restart' : bool # whether the shutdown is final, or precedes a restart
787 787 }
788 788
789 789 .. Note::
790 790
791 791 When the clients detect a dead kernel thanks to inactivity on the heartbeat
792 792 socket, they simply send a forceful process termination signal, since a dead
793 793 process is unlikely to respond in any useful way to messages.
794 794
795 795
796 796 Messages on the PUB/SUB socket
797 797 ==============================
798 798
799 799 Streams (stdout, stderr, etc)
800 800 ------------------------------
801 801
802 802 Message type: ``stream``::
803 803
804 804 content = {
805 805 # The name of the stream is one of 'stdout', 'stderr'
806 806 'name' : str,
807 807
808 808 # The data is an arbitrary string to be written to that stream
809 809 'data' : str,
810 810 }
811 811
812 812 Display Data
813 813 ------------
814 814
815 815 This type of message is used to bring back data that should be diplayed (text,
816 816 html, svg, etc.) in the frontends. This data is published to all frontends.
817 817 Each message can have multiple representations of the data; it is up to the
818 818 frontend to decide which to use and how. A single message should contain all
819 819 possible representations of the same information. Each representation should
820 820 be a JSON'able data structure, and should be a valid MIME type.
821 821
822 822 Some questions remain about this design:
823 823
824 824 * Do we use this message type for pyout/displayhook? Probably not, because
825 825 the displayhook also has to handle the Out prompt display. On the other hand
826 826 we could put that information into the metadata secion.
827 827
828 828 Message type: ``display_data``::
829 829
830 830 content = {
831 831
832 832 # Who create the data
833 833 'source' : str,
834 834
835 835 # The data dict contains key/value pairs, where the kids are MIME
836 836 # types and the values are the raw data of the representation in that
837 837 # format.
838 838 'data' : dict,
839 839
840 840 # Any metadata that describes the data
841 841 'metadata' : dict
842 842 }
843 843
844 844
845 845 The ``metadata`` contains any metadata that describes the output.
846 846 Global keys are assumed to apply to the output as a whole.
847 847 The ``metadata`` dict can also contain mime-type keys, which will be sub-dictionaries,
848 848 which are interpreted as applying only to output of that type.
849 849 Third parties should put any data they write into a single dict
850 850 with a reasonably unique name to avoid conflicts.
851 851
852 852 The only metadata keys currently defined in IPython are the width and height
853 853 of images::
854 854
855 855 'metadata' : {
856 856 'image/png' : {
857 857 'width': 640,
858 858 'height': 480
859 859 }
860 860 }
861 861
862 862
863 863 Raw Data Publication
864 864 --------------------
865 865
866 866 ``display_data`` lets you publish *representations* of data, such as images and html.
867 867 This ``data_pub`` message lets you publish *actual raw data*, sent via message buffers.
868 868
869 869 data_pub messages are constructed via the :func:`IPython.lib.datapub.publish_data` function:
870 870
871 871 .. sourcecode:: python
872 872
873 873 from IPython.kernel.zmq.datapub import publish_data
874 874 ns = dict(x=my_array)
875 875 publish_data(ns)
876 876
877 877
878 878 Message type: ``data_pub``::
879 879
880 880 content = {
881 881 # the keys of the data dict, after it has been unserialized
882 882 keys = ['a', 'b']
883 883 }
884 884 # the namespace dict will be serialized in the message buffers,
885 885 # which will have a length of at least one
886 886 buffers = ['pdict', ...]
887 887
888 888
889 889 The interpretation of a sequence of data_pub messages for a given parent request should be
890 890 to update a single namespace with subsequent results.
891 891
892 892 .. note::
893 893
894 894 No frontends directly handle data_pub messages at this time.
895 895 It is currently only used by the client/engines in :mod:`IPython.parallel`,
896 896 where engines may publish *data* to the Client,
897 897 of which the Client can then publish *representations* via ``display_data``
898 898 to various frontends.
899 899
900 900 Python inputs
901 901 -------------
902 902
903 903 These messages are the re-broadcast of the ``execute_request``.
904 904
905 905 Message type: ``pyin``::
906 906
907 907 content = {
908 908 'code' : str, # Source code to be executed, one or more lines
909 909
910 910 # The counter for this execution is also provided so that clients can
911 911 # display it, since IPython automatically creates variables called _iN
912 912 # (for input prompt In[N]).
913 913 'execution_count' : int
914 914 }
915 915
916 916 Python outputs
917 917 --------------
918 918
919 919 When Python produces output from code that has been compiled in with the
920 920 'single' flag to :func:`compile`, any expression that produces a value (such as
921 921 ``1+1``) is passed to ``sys.displayhook``, which is a callable that can do with
922 922 this value whatever it wants. The default behavior of ``sys.displayhook`` in
923 923 the Python interactive prompt is to print to ``sys.stdout`` the :func:`repr` of
924 924 the value as long as it is not ``None`` (which isn't printed at all). In our
925 925 case, the kernel instantiates as ``sys.displayhook`` an object which has
926 926 similar behavior, but which instead of printing to stdout, broadcasts these
927 927 values as ``pyout`` messages for clients to display appropriately.
928 928
929 929 IPython's displayhook can handle multiple simultaneous formats depending on its
930 930 configuration. The default pretty-printed repr text is always given with the
931 931 ``data`` entry in this message. Any other formats are provided in the
932 932 ``extra_formats`` list. Frontends are free to display any or all of these
933 933 according to its capabilities. ``extra_formats`` list contains 3-tuples of an ID
934 934 string, a type string, and the data. The ID is unique to the formatter
935 935 implementation that created the data. Frontends will typically ignore the ID
936 936 unless if it has requested a particular formatter. The type string tells the
937 937 frontend how to interpret the data. It is often, but not always a MIME type.
938 938 Frontends should ignore types that it does not understand. The data itself is
939 939 any JSON object and depends on the format. It is often, but not always a string.
940 940
941 941 Message type: ``pyout``::
942 942
943 943 content = {
944 944
945 945 # The counter for this execution is also provided so that clients can
946 946 # display it, since IPython automatically creates variables called _N
947 947 # (for prompt N).
948 948 'execution_count' : int,
949 949
950 950 # The data dict contains key/value pairs, where the kids are MIME
951 951 # types and the values are the raw data of the representation in that
952 952 # format. The data dict must minimally contain the ``text/plain``
953 953 # MIME type which is used as a backup representation.
954 954 'data' : dict,
955 955
956 956 }
957 957
958 958 Python errors
959 959 -------------
960 960
961 961 When an error occurs during code execution
962 962
963 963 Message type: ``pyerr``::
964 964
965 965 content = {
966 966 # Similar content to the execute_reply messages for the 'error' case,
967 967 # except the 'status' field is omitted.
968 968 }
969 969
970 970 Kernel status
971 971 -------------
972 972
973 973 This message type is used by frontends to monitor the status of the kernel.
974 974
975 975 Message type: ``status``::
976 976
977 977 content = {
978 978 # When the kernel starts to execute code, it will enter the 'busy'
979 979 # state and when it finishes, it will enter the 'idle' state.
980 980 # The kernel will publish state 'starting' exactly once at process startup.
981 981 execution_state : ('busy', 'idle', 'starting')
982 982 }
983 983
984 984
985 985 Messages on the stdin ROUTER/DEALER sockets
986 986 ===========================================
987 987
988 988 This is a socket where the request/reply pattern goes in the opposite direction:
989 989 from the kernel to a *single* frontend, and its purpose is to allow
990 990 ``raw_input`` and similar operations that read from ``sys.stdin`` on the kernel
991 991 to be fulfilled by the client. The request should be made to the frontend that
992 992 made the execution request that prompted ``raw_input`` to be called. For now we
993 993 will keep these messages as simple as possible, since they only mean to convey
994 994 the ``raw_input(prompt)`` call.
995 995
996 996 Message type: ``input_request``::
997 997
998 998 content = { 'prompt' : str }
999 999
1000 1000 Message type: ``input_reply``::
1001 1001
1002 1002 content = { 'value' : str }
1003 1003
1004 1004 .. Note::
1005 1005
1006 1006 We do not explicitly try to forward the raw ``sys.stdin`` object, because in
1007 1007 practice the kernel should behave like an interactive program. When a
1008 1008 program is opened on the console, the keyboard effectively takes over the
1009 1009 ``stdin`` file descriptor, and it can't be used for raw reading anymore.
1010 1010 Since the IPython kernel effectively behaves like a console program (albeit
1011 1011 one whose "keyboard" is actually living in a separate process and
1012 1012 transported over the zmq connection), raw ``stdin`` isn't expected to be
1013 1013 available.
1014 1014
1015 1015
1016 1016 Heartbeat for kernels
1017 1017 =====================
1018 1018
1019 1019 Initially we had considered using messages like those above over ZMQ for a
1020 1020 kernel 'heartbeat' (a way to detect quickly and reliably whether a kernel is
1021 1021 alive at all, even if it may be busy executing user code). But this has the
1022 1022 problem that if the kernel is locked inside extension code, it wouldn't execute
1023 1023 the python heartbeat code. But it turns out that we can implement a basic
1024 1024 heartbeat with pure ZMQ, without using any Python messaging at all.
1025 1025
1026 1026 The monitor sends out a single zmq message (right now, it is a str of the
1027 1027 monitor's lifetime in seconds), and gets the same message right back, prefixed
1028 1028 with the zmq identity of the DEALER socket in the heartbeat process. This can be
1029 1029 a uuid, or even a full message, but there doesn't seem to be a need for packing
1030 1030 up a message when the sender and receiver are the exact same Python object.
1031 1031
1032 1032 The model is this::
1033 1033
1034 1034 monitor.send(str(self.lifetime)) # '1.2345678910'
1035 1035
1036 1036 and the monitor receives some number of messages of the form::
1037 1037
1038 1038 ['uuid-abcd-dead-beef', '1.2345678910']
1039 1039
1040 1040 where the first part is the zmq.IDENTITY of the heart's DEALER on the engine, and
1041 1041 the rest is the message sent by the monitor. No Python code ever has any
1042 1042 access to the message between the monitor's send, and the monitor's recv.
1043 1043
1044 1044
1045 1045 ToDo
1046 1046 ====
1047 1047
1048 1048 Missing things include:
1049 1049
1050 1050 * Important: finish thinking through the payload concept and API.
1051 1051
1052 1052 * Important: ensure that we have a good solution for magics like %edit. It's
1053 1053 likely that with the payload concept we can build a full solution, but not
1054 1054 100% clear yet.
1055 1055
1056 1056 * Finishing the details of the heartbeat protocol.
1057 1057
1058 1058 * Signal handling: specify what kind of information kernel should broadcast (or
1059 1059 not) when it receives signals.
1060 1060
1061 .. include:: ../links.rst
1061 .. include:: ../links.txt
1 NO CONTENT: file renamed from docs/source/development/parallel_connections.txt to docs/source/development/parallel_connections.rst
1 NO CONTENT: file renamed from docs/source/development/parallel_messages.txt to docs/source/development/parallel_messages.rst
@@ -1,38 +1,38 b''
1 1 =====================
2 2 IPython Documentation
3 3 =====================
4 4
5 5 .. htmlonly::
6 6
7 7 :Release: |release|
8 8 :Date: |today|
9 9
10 10 .. only:: not rtd
11 11
12 12 Welcome to the official IPython documentation.
13 13
14 14 .. only:: rtd
15 15
16 16 This is a partial copy of IPython documentation, please visit `IPython official documentation <http://ipython.org/documentation.html>`_.
17 17
18 18 Contents
19 19 ========
20 20
21 21 .. toctree::
22 22 :maxdepth: 1
23 23
24 overview.txt
25 whatsnew/index.txt
26 install/index.txt
27 interactive/index.txt
28 parallel/index.txt
29 config/index.txt
30 development/index.txt
31 api/index.txt
32 about/index.txt
24 overview
25 whatsnew/index
26 install/index
27 interactive/index
28 parallel/index
29 config/index
30 development/index
31 api/index
32 about/index
33 33
34 34 .. htmlonly::
35 35 * :ref:`genindex`
36 36 * :ref:`modindex`
37 37 * :ref:`search`
38 38
@@ -1,11 +1,11 b''
1 1 .. _install_index:
2 2
3 3 ============
4 4 Installation
5 5 ============
6 6
7 7 .. toctree::
8 8 :maxdepth: 2
9 9
10 install.txt
10 install
11 11
1 NO CONTENT: file renamed from docs/source/install/install.txt to docs/source/install/install.rst
1 NO CONTENT: file renamed from docs/source/interactive/htmlnotebook.txt to docs/source/interactive/htmlnotebook.rst
1 NO CONTENT: file renamed from docs/source/interactive/qtconsole.txt to docs/source/interactive/qtconsole.rst
1 NO CONTENT: file renamed from docs/source/interactive/reference.txt to docs/source/interactive/reference.rst
1 NO CONTENT: file renamed from docs/source/interactive/shell.txt to docs/source/interactive/shell.rst
1 NO CONTENT: file renamed from docs/source/interactive/tips.txt to docs/source/interactive/tips.rst
1 NO CONTENT: file renamed from docs/source/interactive/tutorial.txt to docs/source/interactive/tutorial.rst
1 NO CONTENT: file renamed from docs/source/links.rst to docs/source/links.txt
1 NO CONTENT: file renamed from docs/source/overview.txt to docs/source/overview.rst
1 NO CONTENT: file renamed from docs/source/parallel/asyncresult.txt to docs/source/parallel/asyncresult.rst
1 NO CONTENT: file renamed from docs/source/parallel/dag_dependencies.txt to docs/source/parallel/dag_dependencies.rst
1 NO CONTENT: file renamed from docs/source/parallel/magics.txt to docs/source/parallel/magics.rst
1 NO CONTENT: file renamed from docs/source/parallel/parallel_db.txt to docs/source/parallel/parallel_db.rst
1 NO CONTENT: file renamed from docs/source/parallel/parallel_demos.txt to docs/source/parallel/parallel_demos.rst
1 NO CONTENT: file renamed from docs/source/parallel/parallel_details.txt to docs/source/parallel/parallel_details.rst
1 NO CONTENT: file renamed from docs/source/parallel/parallel_intro.txt to docs/source/parallel/parallel_intro.rst
1 NO CONTENT: file renamed from docs/source/parallel/parallel_mpi.txt to docs/source/parallel/parallel_mpi.rst
1 NO CONTENT: file renamed from docs/source/parallel/parallel_multiengine.txt to docs/source/parallel/parallel_multiengine.rst
1 NO CONTENT: file renamed from docs/source/parallel/parallel_process.txt to docs/source/parallel/parallel_process.rst
1 NO CONTENT: file renamed from docs/source/parallel/parallel_security.txt to docs/source/parallel/parallel_security.rst
1 NO CONTENT: file renamed from docs/source/parallel/parallel_task.txt to docs/source/parallel/parallel_task.rst
1 NO CONTENT: file renamed from docs/source/parallel/parallel_transition.txt to docs/source/parallel/parallel_transition.rst
1 NO CONTENT: file renamed from docs/source/parallel/parallel_winhpc.txt to docs/source/parallel/parallel_winhpc.rst
@@ -1,14 +1,14 b''
1 1 ========================================
2 2 Using IPython on Windows HPC Server 2008
3 3 ========================================
4 4
5 5
6 6 Contents
7 7 ========
8 8
9 9 .. toctree::
10 10 :maxdepth: 1
11 11
12 parallel_winhpc.txt
13 parallel_demos.txt
12 parallel_winhpc
13 parallel_demos
14 14
1 NO CONTENT: file renamed from docs/source/whatsnew/development.txt to docs/source/whatsnew/development.rst
1 NO CONTENT: file renamed from docs/source/whatsnew/github-stats-0.11.txt to docs/source/whatsnew/github-stats-0.11.rst
1 NO CONTENT: file renamed from docs/source/whatsnew/github-stats-0.12.txt to docs/source/whatsnew/github-stats-0.12.rst
1 NO CONTENT: file renamed from docs/source/whatsnew/github-stats-0.13.txt to docs/source/whatsnew/github-stats-0.13.rst
1 NO CONTENT: file renamed from docs/source/whatsnew/github-stats-1.0.txt to docs/source/whatsnew/github-stats-1.0.rst
1 NO CONTENT: file renamed from docs/source/whatsnew/index.txt to docs/source/whatsnew/index.rst
1 NO CONTENT: file renamed from docs/source/whatsnew/version0.10.txt to docs/source/whatsnew/version0.10.rst
1 NO CONTENT: file renamed from docs/source/whatsnew/version0.11.txt to docs/source/whatsnew/version0.11.rst
1 NO CONTENT: file renamed from docs/source/whatsnew/version0.12.txt to docs/source/whatsnew/version0.12.rst
1 NO CONTENT: file renamed from docs/source/whatsnew/version0.13.txt to docs/source/whatsnew/version0.13.rst
1 NO CONTENT: file renamed from docs/source/whatsnew/version0.8.txt to docs/source/whatsnew/version0.8.rst
1 NO CONTENT: file renamed from docs/source/whatsnew/version0.9.txt to docs/source/whatsnew/version0.9.rst
1 NO CONTENT: file renamed from docs/source/whatsnew/version1.0.txt to docs/source/whatsnew/version1.0.rst
1 NO CONTENT: file was removed
1 NO CONTENT: file was removed
1 NO CONTENT: file was removed
General Comments 0
You need to be logged in to leave comments. Login now