Merge pull request #6679 from takluyver/doc-magics

Proper documentation for magics

docs/autogen_magics.py

@@ -0,0 +1,64 b''
+from IPython.core.alias import Alias
+from IPython.core.interactiveshell import InteractiveShell
+from IPython.core.magic import MagicAlias
+from IPython.utils.text import dedent, indent
+
+shell = InteractiveShell.instance()
+magics = shell.magics_manager.magics
+
+def _strip_underline(line):
+    chars = set(line.strip())
+    if len(chars) == 1 and ('-' in chars or '=' in chars):
+        return ""
+    else:
+        return line
+
+def format_docstring(func):
+    docstring = (func.__doc__ or "Undocumented").rstrip()
+    docstring = indent(dedent(docstring))
+    # Sphinx complains if indented bits have rst headings in, so strip out
+    # any underlines in the docstring.
+    lines = [_strip_underline(l) for l in docstring.splitlines()]
+    return "\n".join(lines)
+
+output = [
+"Line magics",
+"===========",
+"",
+]
+
+# Case insensitive sort by name
+def sortkey(s): return s[0].lower()
+
+for name, func in sorted(magics['line'].items(), key=sortkey):
+    if isinstance(func, Alias) or isinstance(func, MagicAlias):
+        # Aliases are magics, but shouldn't be documented here
+        # Also skip aliases to other magics
+        continue
+    output.extend([".. magic:: {}".format(name),
+                   "",
+                   format_docstring(func),
+                   ""])
+
+output.extend([
+"Cell magics",
+"===========",
+"",
+])
+
+for name, func in sorted(magics['cell'].items(), key=sortkey):
+    if name == "!":
+        # Special case - don't encourage people to use %%!
+        continue
+    if func == magics['line'].get(name, 'QQQP'):
+        # Don't redocument line magics that double as cell magics
+        continue
+    if isinstance(func, MagicAlias):
+        continue
+    output.extend([".. cellmagic:: {}".format(name),
+                   "",
+                   format_docstring(func),
+                   ""])
+
+with open("source/interactive/magics-generated.txt", "w") as f:
+    f.write("\n".join(output))
  No newline at end of file

docs/source/interactive/magics.rst

@@ -0,0 +1,5 b''
+=======================
+Built-in magic commands
+=======================
+
+.. include:: magics-generated.txt

docs/sphinxext/magics.py

@@ -0,0 +1,42 b''
+import re
+from sphinx import addnodes
+from sphinx.domains.std import StandardDomain
+from sphinx.roles import XRefRole
+
+name_re = re.compile(r"[\w_]+")
+
+def parse_magic(env, sig, signode):
+    m = name_re.match(sig)
+    if not m:
+        raise Exception("Invalid magic command: %s" % sig)
+    name = "%" + sig
+    signode += addnodes.desc_name(name, name)
+    return m.group(0)
+
+class LineMagicRole(XRefRole):
+    """Cross reference role displayed with a % prefix"""
+    prefix = "%"
+
+    def process_link(self, env, refnode, has_explicit_title, title, target):
+        if not has_explicit_title:
+            title = self.prefix + title.lstrip("%")
+        target = target.lstrip("%")
+        return title, target
+
+def parse_cell_magic(env, sig, signode):
+    m = name_re.match(sig)
+    if not m:
+        raise ValueError("Invalid cell magic: %s" % sig)
+    name = "%%" + sig
+    signode += addnodes.desc_name(name, name)
+    return m.group(0)
+
+class CellMagicRole(LineMagicRole):
+    """Cross reference role displayed with a %% prefix"""
+    prefix = "%%"
+
+def setup(app):
+    app.add_object_type('magic', 'magic', 'pair: %s; magic command', parse_magic)
+    StandardDomain.roles['magic'] = LineMagicRole()
+    app.add_object_type('cellmagic', 'cellmagic', 'pair: %s; cell magic', parse_cell_magic)
+    StandardDomain.roles['cellmagic'] = CellMagicRole()

.gitignore

@@ -5,6 +5,7 b' _build'
 docs/man/*.gz
 docs/source/api/generated
 docs/source/config/options
+docs/source/interactive/magics-generated.txt
 docs/gh-pages
 IPython/html/notebook/static/mathjax
 IPython/html/static/style/*.map

docs/Makefile

@@ -42,6 +42,7 b' clean: clean_api'
 	-rm -rf build/* dist/*
 	-cd $(SRCDIR)/config/options; cat generated | xargs -r rm -f
 	-rm -rf $(SRCDIR)/config/options/generated
+	-rm -f $(SRCDIR)/interactive/magics-generated.txt
 
 pdf: latex
 	cd build/latex && make all-pdf
@@ -58,8 +59,8 b' dist: html'
 	cp -al build/html .
 	@echo "Build finished.  Final docs are in html/"
 
-html: api autoconfig
-html_noapi: clean_api autoconfig
+html: api autoconfig automagic
+html_noapi: clean_api autoconfig automagic
 
 html html_noapi:
 	mkdir -p build/html build/doctrees
@@ -67,6 +68,12 b' html html_noapi:'
 	@echo
 	@echo "Build finished. The HTML pages are in build/html."
 
+automagic: source/interactive/magics-generated.txt
+
+source/interactive/magics-generated.txt: autogen_magics.py
+	python autogen_magics.py
+	@echo "Created docs for line & cell magics"
+
 autoconfig: source/config/options/generated
 
 source/config/options/generated:

docs/autogen_api.py

@@ -25,6 +25,8 b" if __name__ == '__main__':"
                                         r'\.nbformat\.v\d+',
                                         # Public API for this is in kernel.zmq.eventloops
                                         r'\.kernel\.zmq\.gui',
+                                        # Magics are documented separately
+                                        r'\.core\.magics',
                                         ]
 
     # The inputhook* modules often cause problems on import, such as trying to

docs/source/conf.py

@@ -52,6 +52,7 b' extensions = ['
     'IPython.sphinxext.ipython_directive',
     'numpydoc',  # to preprocess docstrings
     'github',  # for easy GitHub links
+    'magics',
 ]
 
 if ON_RTD:

docs/source/interactive/index.rst

@@ -8,6 +8,7 b' Using IPython for interactive work'
    tutorial
    tips
    reference
+   magics
    shell
    qtconsole
 

docs/source/interactive/reference.rst

@@ -88,13 +88,13 b' current execution block. Cell magics can in fact make arbitrary modifications'
 to the input they receive, which need not even be valid Python code at all.
 They receive the whole block as a single string.
 
-As a line magic example, the ``%cd`` magic works just like the OS command of
+As a line magic example, the :magic:`cd` magic works just like the OS command of
 the same name::
 
       In [8]: %cd
       /home/fperez
 
-The following uses the builtin ``timeit`` in cell mode::
+The following uses the builtin :magic:`timeit` in cell mode::
   
   In [10]: %%timeit x = range(10000)
       ...: min(x)
@@ -104,7 +104,7 b' The following uses the builtin ``timeit`` in cell mode::'
 
 In this case, ``x = range(10000)`` is called as the line argument, and the
 block with ``min(x)`` and ``max(x)`` is called as the cell body.  The
-``timeit`` magic receives both.
+:magic:`timeit` magic receives both.
   
 If you have 'automagic' enabled (as it by default), you don't need to type in
 the single ``%`` explicitly for line magics; IPython will scan its internal
@@ -156,6 +156,9 b' docstrings of all currently available magic commands.'
 
 .. seealso::
 
+   :doc:`magics`
+     A list of the line and cell magics available in IPython by default
+
    :ref:`defining_magics`
      How to define and register additional magic functions
 
@@ -185,21 +188,19 b' Typing ``??word`` or ``word??`` gives access to the full information, including'
 the source code where possible. Long strings are not snipped.
 
 The following magic functions are particularly useful for gathering
-information about your working environment. You can get more details by
-typing ``%magic`` or querying them individually (``%function_name?``);
-this is just a summary:
+information about your working environment:
 
-    * **%pdoc <object>**: Print (or run through a pager if too long) the
+    * :magic:`pdoc` **<object>**: Print (or run through a pager if too long) the
       docstring for an object. If the given object is a class, it will
       print both the class and the constructor docstrings.
-    * **%pdef <object>**: Print the call signature for any callable
+    * :magic:`pdef` **<object>**: Print the call signature for any callable
       object. If the object is a class, print the constructor information.
-    * **%psource <object>**: Print (or run through a pager if too long)
+    * :magic:`psource` **<object>**: Print (or run through a pager if too long)
       the source code for an object.
-    * **%pfile <object>**: Show the entire source file where an object was
+    * :magic:`pfile` **<object>**: Show the entire source file where an object was
       defined via a pager, opening it at the line where the object
       definition begins.
-    * **%who/%whos**: These functions give information about identifiers
+    * :magic:`who`/:magic:`whos`: These functions give information about identifiers
       you have defined interactively (not things you loaded or defined
       in your configuration files). %who just prints a list of
       identifiers and %whos prints a table with some basic details about
@@ -310,7 +311,7 b' Session logging and restoring'
 
 You can log all input from a session either by starting IPython with the
 command line switch ``--logfile=foo.py`` (see :ref:`here <command_line_options>`)
-or by activating the logging at any moment with the magic function %logstart.
+or by activating the logging at any moment with the magic function :magic:`logstart`.
 
 Log files can later be reloaded by running them as scripts and IPython
 will attempt to 'replay' the log by executing all the lines in it, thus
@@ -322,7 +323,7 b' any code you wrote while experimenting. Log files are regular text files'
 which you can later open in your favorite text editor to extract code or
 to 'clean them up' before using them to replay a session.
 
-The `%logstart` function for activating logging in mid-session is used as
+The :magic:`logstart` function for activating logging in mid-session is used as
 follows::
 
     %logstart [log_name [log_mode]]
@@ -341,7 +342,7 b' one of (note that the modes are given unquoted):'
     * [append:] well, that says it.
     * [rotate:] create rotating logs log_name.1~, log_name.2~, etc.
 
-The %logoff and %logon functions allow you to temporarily stop and
+The :magic:`logoff` and :magic:`logon` functions allow you to temporarily stop and
 resume logging to a file which had previously been started with
 %logstart. They will fail (with an explanation) if you try to use them
 before logging has been started.
@@ -362,7 +363,7 b' You can assign the result of a system command to a Python variable with the'
 syntax ``myfiles = !ls``. This gets machine readable output from stdout 
 (e.g. without colours), and splits on newlines. To explicitly get this sort of
 output without assigning to a variable, use two exclamation marks (``!!ls``) or
-the ``%sx`` magic command.
+the :magic:`sx` magic command.
 
 The captured list has some convenience features. ``myfiles.n`` or ``myfiles.s``
 returns a string delimited by newlines or spaces, respectively. ``myfiles.p``
@@ -392,7 +393,7 b' Note that `$$` is used to represent a literal `$`.'
 System command aliases
 ----------------------
 
-The %alias magic function allows you to define magic functions which are in fact
+The :magic:`alias` magic function allows you to define magic functions which are in fact
 system shell commands. These aliases can have parameters.
 
 ``%alias alias_name cmd`` defines 'alias_name' as an alias for 'cmd'
@@ -411,10 +412,10 b' replaced by a positional parameter to the call to %parts::'
     In [3]: parts A  
     ERROR: Alias <parts> requires 2 arguments, 1 given.
 
-If called with no parameters, %alias prints the table of currently
+If called with no parameters, :magic:`alias` prints the table of currently
 defined aliases.
 
-The %rehashx magic allows you to load your entire $PATH as
+The :magic:`rehashx` magic allows you to load your entire $PATH as
 ipython aliases. See its docstring for further details.
 
 
@@ -440,7 +441,7 b' detailed tracebacks. Furthermore, both normal and verbose tracebacks can'
 be colored (if your terminal supports it) which makes them much easier
 to parse visually.
 
-See the magic xmode and colors functions for details.
+See the magic :magic:`xmode` and :magic:`colors` functions for details.
 
 These features are basically a terminal version of Ka-Ping Yee's cgitb
 module, now part of the standard Python library.
@@ -454,8 +455,8 b' Input caching system'
 IPython offers numbered prompts (In/Out) with input and output caching
 (also referred to as 'input history'). All input is saved and can be 
 retrieved as variables (besides the usual arrow key recall), in 
-addition to the %rep magic command that brings a history entry
-up for editing on the next  command line.
+addition to the :magic:`rep` magic command that brings a history entry
+up for editing on the next command line.
 
 The following variables always exist:
 
@@ -476,17 +477,17 b' characters. You can also manipulate them like regular variables (they'
 are strings), modify or exec them.
 
 You can also re-execute multiple lines of input easily by using the
-magic %rerun or %macro functions. The macro system also allows you to re-execute
+magic :magic:`rerun` or :magic:`macro` functions. The macro system also allows you to re-execute
 previous lines which include magic function calls (which require special
 processing). Type %macro? for more details on the macro system.
 
-A history function %hist allows you to see any part of your input
+A history function :magic:`history` allows you to see any part of your input
 history by printing a range of the _i variables.
 
 You can also search ('grep') through your history by typing 
 ``%hist -g somestring``. This is handy for searching for URLs, IP addresses,
 etc. You can bring history entries listed by '%hist -g' up for editing
-with the %recall command, or run them immediately with %rerun.
+with the %recall command, or run them immediately with :magic:`rerun`.
 
 .. _output_caching:
 
@@ -522,15 +523,15 b' This system obviously can potentially put heavy memory demands on your'
 system, since it prevents Python's garbage collector from removing any
 previously computed results. You can control how many results are kept
 in memory with the configuration option ``InteractiveShell.cache_size``.
-If you set it to 0, output caching is disabled. You can also use the ``%reset``
-and ``%xdel`` magics to clear large items from memory.
+If you set it to 0, output caching is disabled. You can also use the :magic:`reset`
+and :magic:`xdel` magics to clear large items from memory.
 
 Directory history
 -----------------
 
 Your history of visited directories is kept in the global list _dh, and
-the magic %cd command can be used to go to any entry in that list. The
-%dhist command allows you to view this history. Do ``cd -<TAB>`` to
+the magic :magic:`cd` command can be used to go to any entry in that list. The
+:magic:`dhist` command allows you to view this history. Do ``cd -<TAB>`` to
 conveniently view the directory history.
 
 
@@ -707,7 +708,7 b' allows you to step through code, set breakpoints, watch variables,'
 etc.  IPython makes it very easy to start any script under the control
 of pdb, regardless of whether you have wrapped it into a 'main()'
 function or not. For this, simply type ``%run -d myscript`` at an
-IPython prompt. See the %run command's documentation for more details, including
+IPython prompt. See the :magic:`run` command's documentation for more details, including
 how to control where pdb will stop execution first.
 
 For more information on the use of the pdb debugger, see :ref:`debugger-commands`
@@ -724,9 +725,9 b" while your program is at this point 'dead', all the data is still"
 available and you can walk up and down the stack frame and understand
 the origin of the problem.
 
-You can use the ``%debug`` magic after an exception has occurred to start
+You can use the :magic:`debug` magic after an exception has occurred to start
 post-mortem debugging. IPython can also call debugger every time your code
-triggers an uncaught exception. This feature can be toggled with the %pdb magic
+triggers an uncaught exception. This feature can be toggled with the :magic:`pdb` magic
 command, or you can start IPython with the ``--pdb`` option.
 
 For a post-mortem debugger in your programs outside IPython,
@@ -805,7 +806,7 b' advantages of this are:'
   all of these things.
 
 For users, enabling GUI event loop integration is simple.  You simple use the
-``%gui`` magic as follows::
+:magic:`gui` magic as follows::
 
     %gui [GUINAME]
 
@@ -904,7 +905,7 b' scientific computing, all with a syntax compatible with that of the popular'
 Matlab program.
 
 To start IPython with matplotlib support, use the ``--matplotlib`` switch. If
-IPython is already running, you can run the ``%matplotlib`` magic.  If no
+IPython is already running, you can run the :magic:`matplotlib` magic.  If no
 arguments are given, IPython will automatically detect your choice of
 matplotlib backend.  You can also request a specific backend with
 ``%matplotlib backend``, where ``backend`` must be one of: 'tk', 'qt', 'wx',

docs/source/interactive/tutorial.rst

@@ -60,7 +60,7 b' prefixed with a double ``%%``, and they are functions that get as an argument'
 not only the rest of the line, but also the lines below it in a separate
 argument.
 
-The following examples show how to call the builtin ``timeit`` magic, both in
+The following examples show how to call the builtin :magic:`timeit` magic, both in
 line and cell mode::
 
       In [1]: %timeit range(1000)
@@ -73,19 +73,19 b' line and cell mode::'
 
 The builtin magics include:
 
-- Functions that work with code: ``%run``, ``%edit``, ``%save``, ``%macro``,
-  ``%recall``, etc.
-- Functions which affect the shell: ``%colors``, ``%xmode``, ``%autoindent``,
-  ``%automagic``, etc.
-- Other functions such as ``%reset``, ``%timeit``, ``%%file``, ``%load``, or
-  ``%paste``.
+- Functions that work with code: :magic`run`, :magic:`edit`, :magic:`save`, :magic:`macro`,
+  :magic:`recall`, etc.
+- Functions which affect the shell: :magic:`colors`, :magic:`xmode`, :magic:`autoindent`,
+  :magic:`automagic`, etc.
+- Other functions such as :magic:`reset`, :magic:`timeit`, :cellmagic:`writefile`, :magic:`load`, or
+  :magic:`paste`.
 
 You can always call them using the ``%`` prefix, and if you're calling a line
 magic on a line by itself, you can omit even that::
 
     run thescript.py
 
-You can toggle this behavior by running the ``%automagic`` magic.  Cell magics
+You can toggle this behavior by running the :magic:`automagic` magic.  Cell magics
 must always have the ``%%`` prefix.
 
 A more detailed explanation of the magic system can be obtained by calling
@@ -95,12 +95,14 b' read its docstring. To see all the available magic functions, call'
 
 .. seealso::
 
+    :doc:`magics`
+
     `Cell magics`_ example notebook
 
 Running and Editing
 -------------------
 
-The ``%run`` magic command allows you to run any python script and load all of
+The :magic:`run` magic command allows you to run any python script and load all of
 its data directly into the interactive namespace. Since the file is re-read
 from disk each time, changes you make to it are reflected immediately (unlike
 imported modules, which have to be specifically reloaded). IPython also
@@ -110,15 +112,15 b' includes :ref:`dreload <dreload>`, a recursive reload function.'
 for running them under the control of either Python's pdb debugger (-d) or
 profiler (-p).
 
-The ``%edit`` command gives a reasonable approximation of multiline editing,
+The :magic:`edit` command gives a reasonable approximation of multiline editing,
 by invoking your favorite editor on the spot. IPython will execute the
 code you type in there as if it were typed interactively.
 
 Debugging
 ---------
 
-After an exception occurs, you can call ``%debug`` to jump into the Python
-debugger (pdb) and examine the problem. Alternatively, if you call ``%pdb``,
+After an exception occurs, you can call :magic:`debug` to jump into the Python
+debugger (pdb) and examine the problem. Alternatively, if you call :magic:`pdb`,
 IPython will automatically start the debugger on any uncaught exception. You can
 print variables, see code, execute statements and even walk up and down the
 call stack to track down the true source of the problem. This can be an efficient
@@ -163,7 +165,7 b' You can capture the output into a Python list, e.g.: ``files = !ls``. To pass'
 the values of Python variables or expressions to system commands, prefix them
 with $: ``!grep -rF $pattern ipython/*``. See :ref:`our shell section
 <system_shell_access>` for more details.
-	
+
 Define your own system aliases
 ------------------------------
 
@@ -171,7 +173,7 b" It's convenient to have aliases to the system commands you use most often."
 This allows you to work seamlessly from inside IPython with the same commands
 you are used to in your system shell. IPython comes with some pre-defined
 aliases and a complete system for changing directories, both via a stack (see
-%pushd, %popd and %dhist) and via direct %cd. The latter keeps a history of
+:magic:`pushd`, :magic:`popd` and :magic:`dhist`) and via direct :magic:`cd`. The latter keeps a history of
 visited directories and allows you to go to any previously visited one.