Various improvements to docs infrastructure for magics

docs/Makefile

@@ -70,7 +70,7 html html_noapi:
 
 automagic: source/interactive/magics-generated.txt
 
-source/interactive/magics-generated.txt:
+source/interactive/magics-generated.txt: autogen_magics.py
 	python autogen_magics.py
 	@echo "Created docs for line & cell magics"
 

docs/autogen_magics.py

@@ -11,34 +11,54 def isalias(name):
 line_magics = magic_docs['line']
 cell_magics = magic_docs['cell']
 
+def _strip_underline(line):
+    chars = set(line.strip())
+    if len(chars) == 1 and ('-' in chars or '=' in chars):
+        return ""
+    else:
+        return line
+
+def format_docstring(docstring):
+    docstring = indent(dedent(docstring))
+    # Sphinx complains if indented bits have rst headings in, so strip out
+    # any underlines in the docstring.
+    lines = [_strip_underline(l) for l in docstring.splitlines()]
+    return "\n".join(lines)
+
 output = [
 "Line magics",
 "===========",
 "",
 ]
 
-for name, docstring in sorted(line_magics.items()):
+# Case insensitive sort by name
+def sortkey(s): return s[0].lower()
+
+for name, docstring in sorted(line_magics.items(), key=sortkey):
     if isalias(name):
         # Aliases are magics, but shouldn't be documented here
         continue
-    output.extend([".. magic:: %{}".format(name),
+    output.extend([".. magic:: {}".format(name),
                    "",
-                   indent(dedent(docstring)),
+                   format_docstring(docstring),
                    ""])
 
 output.extend([
-"Cell magics"
-"==========="
+"Cell magics",
+"===========",
 "",
 ])
 
-for name, docstring in sorted(cell_magics.items()):
+for name, docstring in sorted(cell_magics.items(), key=sortkey):
+    if name == "!":
+        # Special case - don't encourage people to use %%!
+        continue
     if docstring == line_magics.get(name, 'QQQP'):
         # Don't redocument line magics that double as cell magics
         continue
-    output.extend([".. cellmagic:: %%{}".format(name),
+    output.extend([".. cellmagic:: {}".format(name),
                    "",
-                   indent(dedent(docstring)),
+                   format_docstring(docstring),
                    ""])
 
 with open("source/interactive/magics-generated.txt", "w") as f:

docs/sphinxext/magics.py

@@ -1,24 +1,42
 import re
 from sphinx import addnodes
+from sphinx.domains.std import StandardDomain
+from sphinx.roles import XRefRole
 
-line_magic_re = re.compile(r"%([\w_]+)")
-cell_magic_re = re.compile(r"%%([\w_]+)")
+name_re = re.compile(r"[\w_]+")
 
 def parse_magic(env, sig, signode):
-    m = line_magic_re.match(sig)
+    m = name_re.match(sig)
     if not m:
         raise Exception("Invalid magic command: %s" % sig)
-    signode += addnodes.desc_name(sig, sig)
-    return m.group(1)
+    name = "%" + sig
+    signode += addnodes.desc_name(name, name)
+    return m.group(0)
+
+class LineMagicRole(XRefRole):
+    """Cross reference role displayed with a % prefix"""
+    prefix = "%"
+
+    def process_link(self, env, refnode, has_explicit_title, title, target):
+        if not has_explicit_title:
+            title = self.prefix + title.lstrip("%")
+        target = target.lstrip("%")
+        return title, target
 
 def parse_cell_magic(env, sig, signode):
-    m = cell_magic_re.match(sig)
+    m = name_re.match(sig)
     if not m:
         raise ValueError("Invalid cell magic: %s" % sig)
-    signode += addnodes.desc_name(sig, sig)
-    return m.group(1)
+    name = "%%" + sig
+    signode += addnodes.desc_name(name, name)
+    return m.group(0)
 
+class CellMagicRole(LineMagicRole):
+    """Cross reference role displayed with a %% prefix"""
+    prefix = "%%"
 
-def setup(app):    
+def setup(app):
     app.add_object_type('magic', 'magic', 'pair: %s; magic command', parse_magic)
+    StandardDomain.roles['magic'] = LineMagicRole()
     app.add_object_type('cellmagic', 'cellmagic', 'pair: %s; cell magic', parse_cell_magic)
+    StandardDomain.roles['cellmagic'] = CellMagicRole()