Merge pull request #3558 from minrk/sphinxext

Add IPython.sphinxext

Install IPython-related sphinx extensions, so that they can be imported rather than other projects needing to ship their own copies.

docs/autogen_api.py

@@ -1,66 +1,66
 #!/usr/bin/env python
 """Script to auto-generate our API docs.
 """
 # stdlib imports
 import os
 import sys
 
 # local imports
 sys.path.append(os.path.abspath('sphinxext'))
 from apigen import ApiDocWriter
 
 #*****************************************************************************
 if __name__ == '__main__':
     pjoin = os.path.join
     package = 'IPython'
     outdir = pjoin('source','api','generated')
     docwriter = ApiDocWriter(package,rst_extension='.txt')
     # You have to escape the . here because . is a special char for regexps.
     # You must do make clean if you change this!
     docwriter.package_skip_patterns += [r'\.fixes$',
                                         r'\.external$',
                                         r'\.extensions',
                                         r'\.kernel\.config',
                                         r'\.attic',
                                         r'\.quarantine',
                                         r'\.deathrow',
                                         r'\.config\.default',
                                         r'\.config\.profile',
                                         r'\.frontend',
                                         r'\.gui',
                                         r'\.kernel',
                                         # For now, the zmq code has
                                         # unconditional top-level code so it's
                                         # not import safe.  This needs fixing
                                         r'\.zmq',
                                         ]
 
     docwriter.module_skip_patterns += [ r'\.core\.fakemodule',
                                         r'\.testing\.iptest',
                                         # Keeping these disabled is OK
                                         r'\.parallel\.controller\.mongodb',
                                         r'\.lib\.inputhookwx',
                                         r'\.lib\.inputhookgtk',
                                         r'\.cocoa',
                                         r'\.ipdoctest',
                                         r'\.Gnuplot',
                                         r'\.frontend\.process\.winprocess',
                                         r'\.frontend',
                                         r'\.Shell',
                                         ]
     
     # If we don't have pexpect, we can't load irunner, so skip any code that
     # depends on it
     try:
         import pexpect
     except ImportError:
         docwriter.module_skip_patterns += [r'\.lib\.irunner',
                                            r'\.testing\.mkdoctests']
     # Now, generate the outputs
     docwriter.write_api_docs(outdir)
     # Write index with .rst extension - we can include it, but Sphinx won't try
     # to compile it
     docwriter.write_index(outdir, 'gen.rst',
                           relative_to = pjoin('source','api')
                           )
-    print '%d files written' % len(docwriter.written_modules)
+    print ('%d files written' % len(docwriter.written_modules))

docs/source/conf.py

@@ -1,228 +1,228
 # -*- coding: utf-8 -*-
 #
 # IPython documentation build configuration file.
 
 # NOTE: This file has been edited manually from the auto-generated one from
 # sphinx.  Do NOT delete and re-generate.  If any changes from sphinx are
 # needed, generate a scratch one and merge by hand any new fields needed.
 
 #
 # This file is execfile()d with the current directory set to its containing dir.
 #
 # The contents of this file are pickled, so don't put values in the namespace
 # that aren't pickleable (module imports are okay, they're removed automatically).
 #
 # All configuration values have a default value; values that are commented out
 # serve to show the default value.
 
 import sys, os
 
 ON_RTD = os.environ.get('READTHEDOCS', None) == 'True'
 
 if ON_RTD:
     # Mock the presence of matplotlib, which we don't have on RTD
     # see
     # http://read-the-docs.readthedocs.org/en/latest/faq.html
     tags.add('rtd')
     
 # If your extensions are in another directory, add it here. If the directory
 # is relative to the documentation root, use os.path.abspath to make it
 # absolute, like shown here.
 sys.path.insert(0, os.path.abspath('../sphinxext'))
 
-# Import support for ipython console session syntax highlighting (lives
-# in the sphinxext directory defined above)
-import ipython_console_highlighting
+# Import support for ipython console session syntax highlighting
+# (lives IPython's sphinxext subpackage)
+from IPython.sphinxext import ipython_console_highlighting
 
 # We load the ipython release info into a dict by explicit execution
 iprelease = {}
 execfile('../../IPython/core/release.py',iprelease)
 
 # General configuration
 # ---------------------
 
 # Add any Sphinx extension module names here, as strings. They can be extensions
 # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
 extensions = [
     'matplotlib.sphinxext.mathmpl',
     'matplotlib.sphinxext.only_directives',
     'matplotlib.sphinxext.plot_directive',
     'sphinx.ext.autodoc',
     'sphinx.ext.doctest',
     'sphinx.ext.inheritance_diagram',
-    'ipython_console_highlighting',
-    'ipython_directive',
+    'IPython.sphinxext.ipython_console_highlighting',
+    'IPython.sphinxext.ipython_directive',
     'numpydoc',  # to preprocess docstrings
     'github',  # for easy GitHub links
 ]
 
 if ON_RTD:
     # Remove extensions not currently supported on RTD
     extensions.remove('matplotlib.sphinxext.only_directives')
     extensions.remove('matplotlib.sphinxext.mathmpl')
     extensions.remove('matplotlib.sphinxext.plot_directive')
-    extensions.remove('ipython_directive')
+    extensions.remove('IPython.sphinxext.ipython_directive')
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
 
 # The suffix of source filenames.
 source_suffix = '.txt'
 
 # The master toctree document.
 master_doc = 'index'
 
 # General substitutions.
 project = 'IPython'
 copyright = '2008, The IPython Development Team'
 
 # ghissue config
 github_project_url = "https://github.com/ipython/ipython"
 
 # The default replacements for |version| and |release|, also used in various
 # other places throughout the built documents.
 #
 # The full version, including alpha/beta/rc tags.
 release = iprelease['version']
 # The short X.Y version.
 version = '.'.join(release.split('.',2)[:2])
 
 
 # There are two options for replacing |today|: either, you set today to some
 # non-false value, then it is used:
 #today = ''
 # Else, today_fmt is used as the format for a strftime call.
 today_fmt = '%B %d, %Y'
 
 # List of documents that shouldn't be included in the build.
 #unused_docs = []
 
 # List of directories, relative to source directories, that shouldn't be searched
 # for source files.
 exclude_dirs = ['attic']
 
 # If true, '()' will be appended to :func: etc. cross-reference text.
 #add_function_parentheses = True
 
 # If true, the current module name will be prepended to all description
 # unit titles (such as .. function::).
 #add_module_names = True
 
 # If true, sectionauthor and moduleauthor directives will be shown in the
 # output. They are ignored by default.
 #show_authors = False
 
 # The name of the Pygments (syntax highlighting) style to use.
 pygments_style = 'sphinx'
 
 
 # Options for HTML output
 # -----------------------
 
 # The style sheet to use for HTML and HTML Help pages. A file of that name
 # must exist either in Sphinx' static/ path, or in one of the custom paths
 # given in html_static_path.
 html_style = 'default.css'
 
 # The name for this set of Sphinx documents.  If None, it defaults to
 # "<project> v<release> documentation".
 #html_title = None
 
 # The name of an image file (within the static path) to place at the top of
 # the sidebar.
 #html_logo = None
 
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
 html_static_path = ['_static']
 
 # If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
 # using the given strftime format.
 html_last_updated_fmt = '%b %d, %Y'
 
 # If true, SmartyPants will be used to convert quotes and dashes to
 # typographically correct entities.
 #html_use_smartypants = True
 
 # Custom sidebar templates, maps document names to template names.
 #html_sidebars = {}
 
 # Additional templates that should be rendered to pages, maps page names to
 # template names.
 #html_additional_pages = {}
 
 # If false, no module index is generated.
 #html_use_modindex = True
 
 # If true, the reST sources are included in the HTML build as _sources/<name>.
 #html_copy_source = True
 
 # If true, an OpenSearch description file will be output, and all pages will
 # contain a <link> tag referring to it.  The value of this option must be the
 # base URL from which the finished HTML is served.
 #html_use_opensearch = ''
 
 # If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml").
 #html_file_suffix = ''
 
 # Output file base name for HTML help builder.
 htmlhelp_basename = 'ipythondoc'
 
 
 # Options for LaTeX output
 # ------------------------
 
 # The paper size ('letter' or 'a4').
 latex_paper_size = 'letter'
 
 # The font size ('10pt', '11pt' or '12pt').
 latex_font_size = '11pt'
 
 # Grouping the document tree into LaTeX files. List of tuples
 # (source start file, target name, title, author, document class [howto/manual]).
 
 latex_documents = [
     ('index', 'ipython.tex', 'IPython Documentation',
      ur"""The IPython Development Team""", 'manual', True),
     ('parallel/winhpc_index', 'winhpc_whitepaper.tex',
      'Using IPython on Windows HPC Server 2008',
      ur"Brian E. Granger", 'manual', True)
 ]
 
 # The name of an image file (relative to this directory) to place at the top of
 # the title page.
 #latex_logo = None
 
 # For "manual" documents, if this is true, then toplevel headings are parts,
 # not chapters.
 #latex_use_parts = False
 
 # Additional stuff for the LaTeX preamble.
 #latex_preamble = ''
 
 # Documents to append as an appendix to all manuals.
 #latex_appendices = []
 
 # If false, no module index is generated.
 latex_use_modindex = True
 
 
 # Options for texinfo output
 # --------------------------
 
 texinfo_documents = [
   (master_doc, 'ipython', 'IPython Documentation',
    'The IPython Development Team',
    'IPython',
    'IPython Documentation',
    'Programming',
    1),
 ]
 
 
 # Cleanup
 # -------
 # delete release info to avoid pickling errors from sphinx
 
 del iprelease

docs/sphinxext/apigen.py

@@ -1,411 +1,413
 """Attempt to generate templates for module reference with Sphinx
 
 XXX - we exclude extension modules
 
 To include extension modules, first identify them as valid in the
 ``_uri2path`` method, then handle them in the ``_parse_module`` script.
 
 We get functions and classes by parsing the text of .py files.
 Alternatively we could import the modules for discovery, and we'd have
 to do that for extension modules.  This would involve changing the
 ``_parse_module`` method to work via import and introspection, and
 might involve changing ``discover_modules`` (which determines which
 files are modules, and therefore which module URIs will be passed to
 ``_parse_module``).
 
 NOTE: this is a modified version of a script originally shipped with the
 PyMVPA project, which we've adapted for NIPY use.  PyMVPA is an MIT-licensed
 project."""
 
+from __future__ import print_function
+
 # Stdlib imports
 import ast
 import os
 import re
 
 class Obj(object):
     '''Namespace to hold arbitrary information.'''
     def __init__(self, **kwargs):
         for k, v in kwargs.items():
             setattr(self, k, v)
                                         
 class FuncClsScanner(ast.NodeVisitor):
     """Scan a module for top-level functions and classes.
     
     Skips objects with an @undoc decorator, or a name starting with '_'.
     """
     def __init__(self):
         ast.NodeVisitor.__init__(self)
         self.classes = []
         self.classes_seen = set()
         self.functions = []
     
     @staticmethod
     def has_undoc_decorator(node):
         return any(isinstance(d, ast.Name) and d.id == 'undoc' \
                                 for d in node.decorator_list)
     
     def visit_FunctionDef(self, node):
         if not (node.name.startswith('_') or self.has_undoc_decorator(node)) \
                 and node.name not in self.functions:
             self.functions.append(node.name)
     
     def visit_ClassDef(self, node):
         if not (node.name.startswith('_') or self.has_undoc_decorator(node)) \
                 and node.name not in self.classes_seen:
             cls = Obj(name=node.name)
             cls.has_init = any(isinstance(n, ast.FunctionDef) and \
                                 n.name=='__init__' for n in node.body)
             self.classes.append(cls)
             self.classes_seen.add(node.name)
     
     def scan(self, mod):
         self.visit(mod)
         return self.functions, self.classes
 
 # Functions and classes
 class ApiDocWriter(object):
     ''' Class for automatic detection and parsing of API docs
     to Sphinx-parsable reST format'''
 
     # only separating first two levels
     rst_section_levels = ['*', '=', '-', '~', '^']
 
     def __init__(self,
                  package_name,
                  rst_extension='.rst',
                  package_skip_patterns=None,
                  module_skip_patterns=None,
                  ):
         ''' Initialize package for parsing
 
         Parameters
         ----------
         package_name : string
             Name of the top-level package.  *package_name* must be the
             name of an importable package
         rst_extension : string, optional
             Extension for reST files, default '.rst'
         package_skip_patterns : None or sequence of {strings, regexps}
             Sequence of strings giving URIs of packages to be excluded
             Operates on the package path, starting at (including) the
             first dot in the package path, after *package_name* - so,
             if *package_name* is ``sphinx``, then ``sphinx.util`` will
             result in ``.util`` being passed for earching by these
             regexps.  If is None, gives default. Default is:
             ['\.tests$']
         module_skip_patterns : None or sequence
             Sequence of strings giving URIs of modules to be excluded
             Operates on the module name including preceding URI path,
             back to the first dot after *package_name*.  For example
             ``sphinx.util.console`` results in the string to search of
             ``.util.console``
             If is None, gives default. Default is:
             ['\.setup$', '\._']
         '''
         if package_skip_patterns is None:
             package_skip_patterns = ['\\.tests$']
         if module_skip_patterns is None:
             module_skip_patterns = ['\\.setup$', '\\._']
         self.package_name = package_name
         self.rst_extension = rst_extension
         self.package_skip_patterns = package_skip_patterns
         self.module_skip_patterns = module_skip_patterns
 
     def get_package_name(self):
         return self._package_name
 
     def set_package_name(self, package_name):
         ''' Set package_name
 
         >>> docwriter = ApiDocWriter('sphinx')
         >>> import sphinx
         >>> docwriter.root_path == sphinx.__path__[0]
         True
         >>> docwriter.package_name = 'docutils'
         >>> import docutils
         >>> docwriter.root_path == docutils.__path__[0]
         True
         '''
         # It's also possible to imagine caching the module parsing here
         self._package_name = package_name
         self.root_module = __import__(package_name)
         self.root_path = self.root_module.__path__[0]
         self.written_modules = None
 
     package_name = property(get_package_name, set_package_name, None,
                             'get/set package_name')
 
     def _uri2path(self, uri):
         ''' Convert uri to absolute filepath
 
         Parameters
         ----------
         uri : string
             URI of python module to return path for
 
         Returns
         -------
         path : None or string
             Returns None if there is no valid path for this URI
             Otherwise returns absolute file system path for URI
 
         Examples
         --------
         >>> docwriter = ApiDocWriter('sphinx')
         >>> import sphinx
         >>> modpath = sphinx.__path__[0]
         >>> res = docwriter._uri2path('sphinx.builder')
         >>> res == os.path.join(modpath, 'builder.py')
         True
         >>> res = docwriter._uri2path('sphinx')
         >>> res == os.path.join(modpath, '__init__.py')
         True
         >>> docwriter._uri2path('sphinx.does_not_exist')
 
         '''
         if uri == self.package_name:
             return os.path.join(self.root_path, '__init__.py')
         path = uri.replace('.', os.path.sep)
         path = path.replace(self.package_name + os.path.sep, '')
         path = os.path.join(self.root_path, path)
         # XXX maybe check for extensions as well?
         if os.path.exists(path + '.py'): # file
             path += '.py'
         elif os.path.exists(os.path.join(path, '__init__.py')):
             path = os.path.join(path, '__init__.py')
         else:
             return None
         return path
 
     def _path2uri(self, dirpath):
         ''' Convert directory path to uri '''
         relpath = dirpath.replace(self.root_path, self.package_name)
         if relpath.startswith(os.path.sep):
             relpath = relpath[1:]
         return relpath.replace(os.path.sep, '.')
 
     def _parse_module(self, uri):
         ''' Parse module defined in *uri* '''
         filename = self._uri2path(uri)
         if filename is None:
             # nothing that we could handle here.
             return ([],[])
         with open(filename, 'rb') as f:
             mod = ast.parse(f.read())
         return FuncClsScanner().scan(mod)
 
     def generate_api_doc(self, uri):
         '''Make autodoc documentation template string for a module
 
         Parameters
         ----------
         uri : string
             python location of module - e.g 'sphinx.builder'
 
         Returns
         -------
         S : string
             Contents of API doc
         '''
         # get the names of all classes and functions
         functions, classes = self._parse_module(uri)
         if not len(functions) and not len(classes):
-            print 'WARNING: Empty -',uri  # dbg
+            print ('WARNING: Empty -', uri)  # dbg
             return ''
 
         # Make a shorter version of the uri that omits the package name for
         # titles
         uri_short = re.sub(r'^%s\.' % self.package_name,'',uri)
 
         ad = '.. AUTO-GENERATED FILE -- DO NOT EDIT!\n\n'
 
         # Set the chapter title to read 'Module:' for all modules except for the
         # main packages
         if '.' in uri:
             chap_title = 'Module: :mod:`' + uri_short + '`'
         else:
             chap_title = ':mod:`' + uri_short + '`'
         ad += chap_title + '\n' + self.rst_section_levels[1] * len(chap_title)
 
         ad += '\n.. automodule:: ' + uri + '\n'
         ad += '\n.. currentmodule:: ' + uri + '\n'
         
         if classes:
             subhead = str(len(classes)) + (' Classes' if len(classes) > 1 else ' Class')
             ad += '\n'+ subhead + '\n' + \
                   self.rst_section_levels[2] * len(subhead) + '\n'
 
         for c in classes:
             ad += '\n.. autoclass:: ' + c.name + '\n'
             # must NOT exclude from index to keep cross-refs working
             ad += '  :members:\n' \
                   '  :show-inheritance:\n'
             if c.has_init:
                   ad += '\n  .. automethod:: __init__\n'
         
         if functions:
             subhead = str(len(functions)) + (' Functions' if len(functions) > 1 else ' Function')
             ad += '\n'+ subhead + '\n' + \
                   self.rst_section_levels[2] * len(subhead) + '\n'
         for f in functions:
             # must NOT exclude from index to keep cross-refs working
             ad += '\n.. autofunction:: ' + uri + '.' + f + '\n\n'
         return ad
 
     def _survives_exclude(self, matchstr, match_type):
         ''' Returns True if *matchstr* does not match patterns
 
         ``self.package_name`` removed from front of string if present
 
         Examples
         --------
         >>> dw = ApiDocWriter('sphinx')
         >>> dw._survives_exclude('sphinx.okpkg', 'package')
         True
         >>> dw.package_skip_patterns.append('^\\.badpkg$')
         >>> dw._survives_exclude('sphinx.badpkg', 'package')
         False
         >>> dw._survives_exclude('sphinx.badpkg', 'module')
         True
         >>> dw._survives_exclude('sphinx.badmod', 'module')
         True
         >>> dw.module_skip_patterns.append('^\\.badmod$')
         >>> dw._survives_exclude('sphinx.badmod', 'module')
         False
         '''
         if match_type == 'module':
             patterns = self.module_skip_patterns
         elif match_type == 'package':
             patterns = self.package_skip_patterns
         else:
             raise ValueError('Cannot interpret match type "%s"'
                              % match_type)
         # Match to URI without package name
         L = len(self.package_name)
         if matchstr[:L] == self.package_name:
             matchstr = matchstr[L:]
         for pat in patterns:
             try:
                 pat.search
             except AttributeError:
                 pat = re.compile(pat)
             if pat.search(matchstr):
                 return False
         return True
 
     def discover_modules(self):
         ''' Return module sequence discovered from ``self.package_name``
 
 
         Parameters
         ----------
         None
 
         Returns
         -------
         mods : sequence
             Sequence of module names within ``self.package_name``
 
         Examples
         --------
         >>> dw = ApiDocWriter('sphinx')
         >>> mods = dw.discover_modules()
         >>> 'sphinx.util' in mods
         True
         >>> dw.package_skip_patterns.append('\.util$')
         >>> 'sphinx.util' in dw.discover_modules()
         False
         >>>
         '''
         modules = [self.package_name]
         # raw directory parsing
         for dirpath, dirnames, filenames in os.walk(self.root_path):
             # Check directory names for packages
             root_uri = self._path2uri(os.path.join(self.root_path,
                                                    dirpath))
             for dirname in dirnames[:]: # copy list - we modify inplace
                 package_uri = '.'.join((root_uri, dirname))
                 if (self._uri2path(package_uri) and
                     self._survives_exclude(package_uri, 'package')):
                     modules.append(package_uri)
                 else:
                     dirnames.remove(dirname)
             # Check filenames for modules
             for filename in filenames:
                 module_name = filename[:-3]
                 module_uri = '.'.join((root_uri, module_name))
                 if (self._uri2path(module_uri) and
                     self._survives_exclude(module_uri, 'module')):
                     modules.append(module_uri)
         return sorted(modules)
 
     def write_modules_api(self, modules,outdir):
         # write the list
         written_modules = []
         for m in modules:
             api_str = self.generate_api_doc(m)
             if not api_str:
                 continue
             # write out to file
             outfile = os.path.join(outdir,
                                    m + self.rst_extension)
             fileobj = open(outfile, 'wt')
             fileobj.write(api_str)
             fileobj.close()
             written_modules.append(m)
         self.written_modules = written_modules
 
     def write_api_docs(self, outdir):
         """Generate API reST files.
 
         Parameters
         ----------
         outdir : string
             Directory name in which to store files
             We create automatic filenames for each module
 
         Returns
         -------
         None
 
         Notes
         -----
         Sets self.written_modules to list of written modules
         """
         if not os.path.exists(outdir):
             os.mkdir(outdir)
         # compose list of modules
         modules = self.discover_modules()
         self.write_modules_api(modules,outdir)
 
     def write_index(self, outdir, path='gen.rst', relative_to=None):
         """Make a reST API index file from written files
 
         Parameters
         ----------
         outdir : string
             Directory to which to write generated index file
         path : string
             Filename to write index to
         relative_to : string
             path to which written filenames are relative.  This
             component of the written file path will be removed from
             outdir, in the generated index.  Default is None, meaning,
             leave path as it is.
         """
         if self.written_modules is None:
             raise ValueError('No modules written')
         # Get full filename path
         path = os.path.join(outdir, path)
         # Path written into index is relative to rootpath
         if relative_to is not None:
             relpath = outdir.replace(relative_to + os.path.sep, '')
         else:
             relpath = outdir
         idx = open(path,'wt')
         w = idx.write
         w('.. AUTO-GENERATED FILE -- DO NOT EDIT!\n\n')
         w('.. toctree::\n\n')
         for f in self.written_modules:
             w('   %s\n' % os.path.join(relpath,f))
         idx.close()