Add option to generate API docs based on __all__

IPython/nbformat/current.py

@@ -1,217 +1,223 b''
 """The official API for working with notebooks in the current format version."""
 
 from __future__ import print_function
 
 from xml.etree import ElementTree as ET
 import re
 
 from IPython.utils.py3compat import unicode_type
 
 from IPython.nbformat.v3 import (
     NotebookNode,
     new_code_cell, new_text_cell, new_notebook, new_output, new_worksheet,
     parse_filename, new_metadata, new_author, new_heading_cell, nbformat,
     nbformat_minor, nbformat_schema, to_notebook_json
 )
 from IPython.nbformat import v3 as _v_latest
 
 from .reader import reads as reader_reads
 from .reader import versions
 from .convert import convert
 from .validator import validate
 
 from IPython.utils.log import get_logger
 
+__all__ = ['NotebookNode', 'new_code_cell', 'new_text_cell', 'new_notebook',
+'new_output', 'new_worksheet', 'parse_filename', 'new_metadata', 'new_author',
+'new_heading_cell', 'nbformat', 'nbformat_minor', 'nbformat_schema',
+'to_notebook_json', 'convert', 'validate', 'NBFormatError', 'parse_py',
+'reads_json', 'writes_json', 'reads_py', 'writes_py', 'reads', 'writes', 'read',
+'write']
 
 current_nbformat = nbformat
 current_nbformat_minor = nbformat_minor
 current_nbformat_module = _v_latest.__name__
 
 
 def docstring_nbformat_mod(func):
     """Decorator for docstrings referring to classes/functions accessed through
     nbformat.current.
 
     Put {nbformat_mod} in the docstring in place of 'IPython.nbformat.v3'.
     """
     func.__doc__ = func.__doc__.format(nbformat_mod=current_nbformat_module)
     return func
 
 
 class NBFormatError(ValueError):
     pass
 
 
 def parse_py(s, **kwargs):
     """Parse a string into a (nbformat, string) tuple."""
     nbf = current_nbformat
     nbm = current_nbformat_minor
     
     pattern = r'# <nbformat>(?P<nbformat>\d+[\.\d+]*)</nbformat>'
     m = re.search(pattern,s)
     if m is not None:
         digits = m.group('nbformat').split('.')
         nbf = int(digits[0])
         if len(digits) > 1:
             nbm = int(digits[1])
 
     return nbf, nbm, s
 
 
 def reads_json(nbjson, **kwargs):
     """Read a JSON notebook from a string and return the NotebookNode
     object. Report if any JSON format errors are detected.
 
     """
     nb = reader_reads(nbjson, **kwargs)
     nb_current = convert(nb, current_nbformat)
     errors = validate(nb_current)
     if errors:
         get_logger().error(
             "Notebook JSON is invalid (%d errors detected during read)",
             len(errors))
     return nb_current
 
 
 def writes_json(nb, **kwargs):
     """Take a NotebookNode object and write out a JSON string. Report if
     any JSON format errors are detected.
 
     """
     errors = validate(nb)
     if errors:
         get_logger().error(
             "Notebook JSON is invalid (%d errors detected during write)",
             len(errors))
     nbjson = versions[current_nbformat].writes_json(nb, **kwargs)
     return nbjson
 
 
 def reads_py(s, **kwargs):
     """Read a .py notebook from a string and return the NotebookNode object."""
     nbf, nbm, s = parse_py(s, **kwargs)
     if nbf in (2, 3):
         nb = versions[nbf].to_notebook_py(s, **kwargs)
     else:
         raise NBFormatError('Unsupported PY nbformat version: %i' % nbf)
     return nb
 
 
 def writes_py(nb, **kwargs):
     # nbformat 3 is the latest format that supports py
     return versions[3].writes_py(nb, **kwargs)
 
 
 # High level API
 
 
 def reads(s, format, **kwargs):
     """Read a notebook from a string and return the NotebookNode object.
 
     This function properly handles notebooks of any version. The notebook
     returned will always be in the current version's format.
 
     Parameters
     ----------
     s : unicode
         The raw unicode string to read the notebook from.
     format : (u'json', u'ipynb', u'py')
         The format that the string is in.
 
     Returns
     -------
     nb : NotebookNode
         The notebook that was read.
     """
     format = unicode_type(format)
     if format == u'json' or format == u'ipynb':
         return reads_json(s, **kwargs)
     elif format == u'py':
         return reads_py(s, **kwargs)
     else:
         raise NBFormatError('Unsupported format: %s' % format)
 
 
 def writes(nb, format, **kwargs):
     """Write a notebook to a string in a given format in the current nbformat version.
 
     This function always writes the notebook in the current nbformat version.
 
     Parameters
     ----------
     nb : NotebookNode
         The notebook to write.
     format : (u'json', u'ipynb', u'py')
         The format to write the notebook in.
 
     Returns
     -------
     s : unicode
         The notebook string.
     """
     format = unicode_type(format)
     if format == u'json' or format == u'ipynb':
         return writes_json(nb, **kwargs)
     elif format == u'py':
         return writes_py(nb, **kwargs)
     else:
         raise NBFormatError('Unsupported format: %s' % format)
 
 
 def read(fp, format, **kwargs):
     """Read a notebook from a file and return the NotebookNode object.
 
     This function properly handles notebooks of any version. The notebook
     returned will always be in the current version's format.
 
     Parameters
     ----------
     fp : file
         Any file-like object with a read method.
     format : (u'json', u'ipynb', u'py')
         The format that the string is in.
 
     Returns
     -------
     nb : NotebookNode
         The notebook that was read.
     """
     return reads(fp.read(), format, **kwargs)
 
 
 def write(nb, fp, format, **kwargs):
     """Write a notebook to a file in a given format in the current nbformat version.
 
     This function always writes the notebook in the current nbformat version.
 
     Parameters
     ----------
     nb : NotebookNode
         The notebook to write.
     fp : file
         Any file-like object with a write method.
     format : (u'json', u'ipynb', u'py')
         The format to write the notebook in.
 
     Returns
     -------
     s : unicode
         The notebook string.
     """
     return fp.write(writes(nb, format, **kwargs))
 
 def _convert_to_metadata():
     """Convert to a notebook having notebook metadata."""
     import glob
     for fname in glob.glob('*.ipynb'):
         print('Converting file:',fname)
         with open(fname,'r') as f:
             nb = read(f,u'json')
         md = new_metadata()
         if u'name' in nb:
             md.name = nb.name
             del nb[u'name']            
         nb.metadata = md
         with open(fname,'w') as f:
             write(nb, f, u'json')
 

docs/autogen_api.py

@@ -1,45 +1,58 b''
 #!/usr/bin/env python
 """Script to auto-generate our API docs.
 """
 # stdlib imports
 import os
 import sys
 
 # local imports
 sys.path.append(os.path.abspath('sphinxext'))
 from apigen import ApiDocWriter
 
 #*****************************************************************************
 if __name__ == '__main__':
     pjoin = os.path.join
     package = 'IPython'
     outdir = pjoin('source','api','generated')
     docwriter = ApiDocWriter(package,rst_extension='.rst')
     # You have to escape the . here because . is a special char for regexps.
     # You must do make clean if you change this!
     docwriter.package_skip_patterns += [r'\.external$',
                                         # Extensions are documented elsewhere.
                                         r'\.extensions',
                                         r'\.config\.profile',
+                                        # These should be accessed via nbformat.current
+                                        r'\.nbformat\.v\d+',
                                         ]
 
     # The inputhook* modules often cause problems on import, such as trying to
     # load incompatible Qt bindings. It's easiest to leave them all out. The
     # main API is in the inputhook module, which is documented.
     docwriter.module_skip_patterns += [ r'\.lib\.inputhook.+',
                                         r'\.ipdoctest',
                                         r'\.testing\.plugin',
                                         # This just prints a deprecation msg:
                                         r'\.frontend$',
                                         # We document this manually.
                                         r'\.utils\.py3compat',
+                                        # These are exposed by nbformat.current
+                                        r'\.nbformat\.convert',
+                                        r'\.nbformat\.validator',
                                         ]
+
+    # These modules import functions and classes from other places to expose
+    # them as part of the public API. They must have __all__ defined. The
+    # non-API modules they import from should be excluded by the skip patterns
+    # above.
+    docwriter.names_from__all__.update({
+        'IPython.nbformat.current',
+    })
     
     # Now, generate the outputs
     docwriter.write_api_docs(outdir)
     # Write index with .txt extension - we can include it, but Sphinx won't try
     # to compile it
     docwriter.write_index(outdir, 'gen.txt',
                           relative_to = pjoin('source','api')
                           )
     print ('%d files written' % len(docwriter.written_modules))

docs/sphinxext/apigen.py

@@ -1,422 +1,454 b''
 """Attempt to generate templates for module reference with Sphinx
 
 XXX - we exclude extension modules
 
 To include extension modules, first identify them as valid in the
 ``_uri2path`` method, then handle them in the ``_parse_module`` script.
 
 We get functions and classes by parsing the text of .py files.
 Alternatively we could import the modules for discovery, and we'd have
 to do that for extension modules.  This would involve changing the
 ``_parse_module`` method to work via import and introspection, and
 might involve changing ``discover_modules`` (which determines which
 files are modules, and therefore which module URIs will be passed to
 ``_parse_module``).
 
 NOTE: this is a modified version of a script originally shipped with the
 PyMVPA project, which we've adapted for NIPY use.  PyMVPA is an MIT-licensed
 project."""
 
 from __future__ import print_function
 
 # Stdlib imports
 import ast
+import inspect
 import os
 import re
 
 class Obj(object):
     '''Namespace to hold arbitrary information.'''
     def __init__(self, **kwargs):
         for k, v in kwargs.items():
             setattr(self, k, v)
                                         
 class FuncClsScanner(ast.NodeVisitor):
     """Scan a module for top-level functions and classes.
     
     Skips objects with an @undoc decorator, or a name starting with '_'.
     """
     def __init__(self):
         ast.NodeVisitor.__init__(self)
         self.classes = []
         self.classes_seen = set()
         self.functions = []
     
     @staticmethod
     def has_undoc_decorator(node):
         return any(isinstance(d, ast.Name) and d.id == 'undoc' \
                                 for d in node.decorator_list)
 
     def visit_If(self, node):
         if isinstance(node.test, ast.Compare) \
                 and isinstance(node.test.left, ast.Name) \
                 and node.test.left.id == '__name__':
             return   # Ignore classes defined in "if __name__ == '__main__':"
 
         self.generic_visit(node)
     
     def visit_FunctionDef(self, node):
         if not (node.name.startswith('_') or self.has_undoc_decorator(node)) \
                 and node.name not in self.functions:
             self.functions.append(node.name)
     
     def visit_ClassDef(self, node):
         if not (node.name.startswith('_') or self.has_undoc_decorator(node)) \
                 and node.name not in self.classes_seen:
             cls = Obj(name=node.name)
             cls.has_init = any(isinstance(n, ast.FunctionDef) and \
                                 n.name=='__init__' for n in node.body)
             self.classes.append(cls)
             self.classes_seen.add(node.name)
     
     def scan(self, mod):
         self.visit(mod)
         return self.functions, self.classes
 
 # Functions and classes
 class ApiDocWriter(object):
     ''' Class for automatic detection and parsing of API docs
     to Sphinx-parsable reST format'''
 
     # only separating first two levels
     rst_section_levels = ['*', '=', '-', '~', '^']
 
     def __init__(self,
                  package_name,
                  rst_extension='.rst',
                  package_skip_patterns=None,
                  module_skip_patterns=None,
+                 names_from__all__=None,
                  ):
         ''' Initialize package for parsing
 
         Parameters
         ----------
         package_name : string
             Name of the top-level package.  *package_name* must be the
             name of an importable package
         rst_extension : string, optional
             Extension for reST files, default '.rst'
         package_skip_patterns : None or sequence of {strings, regexps}
             Sequence of strings giving URIs of packages to be excluded
             Operates on the package path, starting at (including) the
             first dot in the package path, after *package_name* - so,
             if *package_name* is ``sphinx``, then ``sphinx.util`` will
             result in ``.util`` being passed for earching by these
             regexps.  If is None, gives default. Default is:
             ['\.tests$']
         module_skip_patterns : None or sequence
             Sequence of strings giving URIs of modules to be excluded
             Operates on the module name including preceding URI path,
             back to the first dot after *package_name*.  For example
             ``sphinx.util.console`` results in the string to search of
             ``.util.console``
             If is None, gives default. Default is:
             ['\.setup$', '\._']
+        names_from__all__ : set, optional
+            Modules listed in here will be scanned by doing ``from mod import *``,
+            rather than finding function and class definitions by scanning the
+            AST. This is intended for API modules which expose things defined in
+            other files. Modules listed here must define ``__all__`` to avoid
+            exposing everything they import.
         '''
         if package_skip_patterns is None:
             package_skip_patterns = ['\\.tests$']
         if module_skip_patterns is None:
             module_skip_patterns = ['\\.setup$', '\\._']
         self.package_name = package_name
         self.rst_extension = rst_extension
         self.package_skip_patterns = package_skip_patterns
         self.module_skip_patterns = module_skip_patterns
+        self.names_from__all__ = names_from__all__ or set()
 
     def get_package_name(self):
         return self._package_name
 
     def set_package_name(self, package_name):
         ''' Set package_name
 
         >>> docwriter = ApiDocWriter('sphinx')
         >>> import sphinx
         >>> docwriter.root_path == sphinx.__path__[0]
         True
         >>> docwriter.package_name = 'docutils'
         >>> import docutils
         >>> docwriter.root_path == docutils.__path__[0]
         True
         '''
         # It's also possible to imagine caching the module parsing here
         self._package_name = package_name
         self.root_module = __import__(package_name)
         self.root_path = self.root_module.__path__[0]
         self.written_modules = None
 
     package_name = property(get_package_name, set_package_name, None,
                             'get/set package_name')
 
     def _uri2path(self, uri):
         ''' Convert uri to absolute filepath
 
         Parameters
         ----------
         uri : string
             URI of python module to return path for
 
         Returns
         -------
         path : None or string
             Returns None if there is no valid path for this URI
             Otherwise returns absolute file system path for URI
 
         Examples
         --------
         >>> docwriter = ApiDocWriter('sphinx')
         >>> import sphinx
         >>> modpath = sphinx.__path__[0]
         >>> res = docwriter._uri2path('sphinx.builder')
         >>> res == os.path.join(modpath, 'builder.py')
         True
         >>> res = docwriter._uri2path('sphinx')
         >>> res == os.path.join(modpath, '__init__.py')
         True
         >>> docwriter._uri2path('sphinx.does_not_exist')
 
         '''
         if uri == self.package_name:
             return os.path.join(self.root_path, '__init__.py')
         path = uri.replace('.', os.path.sep)
         path = path.replace(self.package_name + os.path.sep, '')
         path = os.path.join(self.root_path, path)
         # XXX maybe check for extensions as well?
         if os.path.exists(path + '.py'): # file
             path += '.py'
         elif os.path.exists(os.path.join(path, '__init__.py')):
             path = os.path.join(path, '__init__.py')
         else:
             return None
         return path
 
     def _path2uri(self, dirpath):
         ''' Convert directory path to uri '''
         relpath = dirpath.replace(self.root_path, self.package_name)
         if relpath.startswith(os.path.sep):
             relpath = relpath[1:]
         return relpath.replace(os.path.sep, '.')
 
     def _parse_module(self, uri):
         ''' Parse module defined in *uri* '''
         filename = self._uri2path(uri)
         if filename is None:
             # nothing that we could handle here.
             return ([],[])
         with open(filename, 'rb') as f:
             mod = ast.parse(f.read())
         return FuncClsScanner().scan(mod)
 
+    def _import_funcs_classes(self, uri):
+        """Import * from uri, and separate out functions and classes."""
+        ns = {}
+        exec('from %s import *' % uri, ns)
+        funcs, classes = [], []
+        for name, obj in ns.items():
+            if inspect.isclass(obj):
+                cls = Obj(name=name, has_init='__init__' in obj.__dict__)
+                classes.append(cls)
+            elif inspect.isfunction(obj):
+                funcs.append(name)
+
+        return sorted(funcs), sorted(classes, key=lambda x: x.name)
+
+    def find_funcs_classes(self, uri):
+        """Find the functions and classes defined in the module ``uri``"""
+        if uri in self.names_from__all__:
+            # For API modules which expose things defined elsewhere, import them
+            return self._import_funcs_classes(uri)
+        else:
+            # For other modules, scan their AST to see what they define
+            return self._parse_module(uri)
+
     def generate_api_doc(self, uri):
         '''Make autodoc documentation template string for a module
 
         Parameters
         ----------
         uri : string
             python location of module - e.g 'sphinx.builder'
 
         Returns
         -------
         S : string
             Contents of API doc
         '''
         # get the names of all classes and functions
-        functions, classes = self._parse_module(uri)
+        functions, classes = self.find_funcs_classes(uri)
         if not len(functions) and not len(classes):
             #print ('WARNING: Empty -', uri)  # dbg
             return ''
 
         # Make a shorter version of the uri that omits the package name for
         # titles
         uri_short = re.sub(r'^%s\.' % self.package_name,'',uri)
 
         ad = '.. AUTO-GENERATED FILE -- DO NOT EDIT!\n\n'
 
         # Set the chapter title to read 'Module:' for all modules except for the
         # main packages
         if '.' in uri:
             chap_title = 'Module: :mod:`' + uri_short + '`'
         else:
             chap_title = ':mod:`' + uri_short + '`'
         ad += chap_title + '\n' + self.rst_section_levels[1] * len(chap_title)
 
         ad += '\n.. automodule:: ' + uri + '\n'
         ad += '\n.. currentmodule:: ' + uri + '\n'
         
         if classes:
             subhead = str(len(classes)) + (' Classes' if len(classes) > 1 else ' Class')
             ad += '\n'+ subhead + '\n' + \
                   self.rst_section_levels[2] * len(subhead) + '\n'
 
         for c in classes:
             ad += '\n.. autoclass:: ' + c.name + '\n'
             # must NOT exclude from index to keep cross-refs working
             ad += '  :members:\n' \
                   '  :show-inheritance:\n'
             if c.has_init:
                   ad += '\n  .. automethod:: __init__\n'
         
         if functions:
             subhead = str(len(functions)) + (' Functions' if len(functions) > 1 else ' Function')
             ad += '\n'+ subhead + '\n' + \
                   self.rst_section_levels[2] * len(subhead) + '\n'
         for f in functions:
             # must NOT exclude from index to keep cross-refs working
             ad += '\n.. autofunction:: ' + uri + '.' + f + '\n\n'
         return ad
 
     def _survives_exclude(self, matchstr, match_type):
         ''' Returns True if *matchstr* does not match patterns
 
         ``self.package_name`` removed from front of string if present
 
         Examples
         --------
         >>> dw = ApiDocWriter('sphinx')
         >>> dw._survives_exclude('sphinx.okpkg', 'package')
         True
         >>> dw.package_skip_patterns.append('^\\.badpkg$')
         >>> dw._survives_exclude('sphinx.badpkg', 'package')
         False
         >>> dw._survives_exclude('sphinx.badpkg', 'module')
         True
         >>> dw._survives_exclude('sphinx.badmod', 'module')
         True
         >>> dw.module_skip_patterns.append('^\\.badmod$')
         >>> dw._survives_exclude('sphinx.badmod', 'module')
         False
         '''
         if match_type == 'module':
             patterns = self.module_skip_patterns
         elif match_type == 'package':
             patterns = self.package_skip_patterns
         else:
             raise ValueError('Cannot interpret match type "%s"'
                              % match_type)
         # Match to URI without package name
         L = len(self.package_name)
         if matchstr[:L] == self.package_name:
             matchstr = matchstr[L:]
         for pat in patterns:
             try:
                 pat.search
             except AttributeError:
                 pat = re.compile(pat)
             if pat.search(matchstr):
                 return False
         return True
 
     def discover_modules(self):
         ''' Return module sequence discovered from ``self.package_name``
 
 
         Parameters
         ----------
         None
 
         Returns
         -------
         mods : sequence
             Sequence of module names within ``self.package_name``
 
         Examples
         --------
         >>> dw = ApiDocWriter('sphinx')
         >>> mods = dw.discover_modules()
         >>> 'sphinx.util' in mods
         True
         >>> dw.package_skip_patterns.append('\.util$')
         >>> 'sphinx.util' in dw.discover_modules()
         False
         >>>
         '''
         modules = [self.package_name]
         # raw directory parsing
         for dirpath, dirnames, filenames in os.walk(self.root_path):
             # Check directory names for packages
             root_uri = self._path2uri(os.path.join(self.root_path,
                                                    dirpath))
             for dirname in dirnames[:]: # copy list - we modify inplace
                 package_uri = '.'.join((root_uri, dirname))
                 if (self._uri2path(package_uri) and
                     self._survives_exclude(package_uri, 'package')):
                     modules.append(package_uri)
                 else:
                     dirnames.remove(dirname)
             # Check filenames for modules
             for filename in filenames:
                 module_name = filename[:-3]
                 module_uri = '.'.join((root_uri, module_name))
                 if (self._uri2path(module_uri) and
                     self._survives_exclude(module_uri, 'module')):
                     modules.append(module_uri)
         return sorted(modules)
 
     def write_modules_api(self, modules,outdir):
         # write the list
         written_modules = []
         for m in modules:
             api_str = self.generate_api_doc(m)
             if not api_str:
                 continue
             # write out to file
             outfile = os.path.join(outdir,
                                    m + self.rst_extension)
             fileobj = open(outfile, 'wt')
             fileobj.write(api_str)
             fileobj.close()
             written_modules.append(m)
         self.written_modules = written_modules
 
     def write_api_docs(self, outdir):
         """Generate API reST files.
 
         Parameters
         ----------
         outdir : string
             Directory name in which to store files
             We create automatic filenames for each module
 
         Returns
         -------
         None
 
         Notes
         -----
         Sets self.written_modules to list of written modules
         """
         if not os.path.exists(outdir):
             os.mkdir(outdir)
         # compose list of modules
         modules = self.discover_modules()
         self.write_modules_api(modules,outdir)
 
     def write_index(self, outdir, path='gen.rst', relative_to=None):
         """Make a reST API index file from written files
 
         Parameters
         ----------
         outdir : string
             Directory to which to write generated index file
         path : string
             Filename to write index to
         relative_to : string
             path to which written filenames are relative.  This
             component of the written file path will be removed from
             outdir, in the generated index.  Default is None, meaning,
             leave path as it is.
         """
         if self.written_modules is None:
             raise ValueError('No modules written')
         # Get full filename path
         path = os.path.join(outdir, path)
         # Path written into index is relative to rootpath
         if relative_to is not None:
             relpath = outdir.replace(relative_to + os.path.sep, '')
         else:
             relpath = outdir
         idx = open(path,'wt')
         w = idx.write
         w('.. AUTO-GENERATED FILE -- DO NOT EDIT!\n\n')
         w('.. autosummary::\n'
           '   :toctree: %s\n\n' % relpath)
         for mod in self.written_modules:
             w('   %s\n' % mod)
         idx.close()