|
|
"""Attempt to generate templates for module reference with Sphinx
|
|
|
|
|
|
XXX - we exclude extension modules
|
|
|
|
|
|
To include extension modules, first identify them as valid in the
|
|
|
``_uri2path`` method, then handle them in the ``_parse_module`` script.
|
|
|
|
|
|
We get functions and classes by parsing the text of .py files.
|
|
|
Alternatively we could import the modules for discovery, and we'd have
|
|
|
to do that for extension modules. This would involve changing the
|
|
|
``_parse_module`` method to work via import and introspection, and
|
|
|
might involve changing ``discover_modules`` (which determines which
|
|
|
files are modules, and therefore which module URIs will be passed to
|
|
|
``_parse_module``).
|
|
|
|
|
|
NOTE: this is a modified version of a script originally shipped with the
|
|
|
PyMVPA project, which we've adapted for NIPY use. PyMVPA is an MIT-licensed
|
|
|
project."""
|
|
|
|
|
|
|
|
|
# Stdlib imports
|
|
|
import ast
|
|
|
import inspect
|
|
|
import os
|
|
|
import re
|
|
|
from importlib import import_module
|
|
|
from types import SimpleNamespace as Obj
|
|
|
|
|
|
|
|
|
class FuncClsScanner(ast.NodeVisitor):
|
|
|
"""Scan a module for top-level functions and classes.
|
|
|
|
|
|
Skips objects with an @undoc decorator, or a name starting with '_'.
|
|
|
"""
|
|
|
def __init__(self):
|
|
|
ast.NodeVisitor.__init__(self)
|
|
|
self.classes = []
|
|
|
self.classes_seen = set()
|
|
|
self.functions = []
|
|
|
|
|
|
@staticmethod
|
|
|
def has_undoc_decorator(node):
|
|
|
return any(isinstance(d, ast.Name) and d.id == 'undoc' \
|
|
|
for d in node.decorator_list)
|
|
|
|
|
|
def visit_If(self, node):
|
|
|
if isinstance(node.test, ast.Compare) \
|
|
|
and isinstance(node.test.left, ast.Name) \
|
|
|
and node.test.left.id == '__name__':
|
|
|
return # Ignore classes defined in "if __name__ == '__main__':"
|
|
|
|
|
|
self.generic_visit(node)
|
|
|
|
|
|
def visit_FunctionDef(self, node):
|
|
|
if not (node.name.startswith('_') or self.has_undoc_decorator(node)) \
|
|
|
and node.name not in self.functions:
|
|
|
self.functions.append(node.name)
|
|
|
|
|
|
def visit_ClassDef(self, node):
|
|
|
if (
|
|
|
not (node.name.startswith("_") or self.has_undoc_decorator(node))
|
|
|
and node.name not in self.classes_seen
|
|
|
):
|
|
|
cls = Obj(name=node.name, sphinx_options={})
|
|
|
cls.has_init = any(
|
|
|
isinstance(n, ast.FunctionDef) and n.name == "__init__"
|
|
|
for n in node.body
|
|
|
)
|
|
|
self.classes.append(cls)
|
|
|
self.classes_seen.add(node.name)
|
|
|
|
|
|
def scan(self, mod):
|
|
|
self.visit(mod)
|
|
|
return self.functions, self.classes
|
|
|
|
|
|
# Functions and classes
|
|
|
class ApiDocWriter(object):
|
|
|
''' Class for automatic detection and parsing of API docs
|
|
|
to Sphinx-parsable reST format'''
|
|
|
|
|
|
# only separating first two levels
|
|
|
rst_section_levels = ['*', '=', '-', '~', '^']
|
|
|
|
|
|
def __init__(self,
|
|
|
package_name,
|
|
|
rst_extension='.rst',
|
|
|
package_skip_patterns=None,
|
|
|
module_skip_patterns=None,
|
|
|
names_from__all__=None,
|
|
|
):
|
|
|
''' Initialize package for parsing
|
|
|
|
|
|
Parameters
|
|
|
----------
|
|
|
package_name : string
|
|
|
Name of the top-level package. *package_name* must be the
|
|
|
name of an importable package
|
|
|
rst_extension : string, optional
|
|
|
Extension for reST files, default '.rst'
|
|
|
package_skip_patterns : None or sequence of {strings, regexps}
|
|
|
Sequence of strings giving URIs of packages to be excluded
|
|
|
Operates on the package path, starting at (including) the
|
|
|
first dot in the package path, after *package_name* - so,
|
|
|
if *package_name* is ``sphinx``, then ``sphinx.util`` will
|
|
|
result in ``.util`` being passed for earching by these
|
|
|
regexps. If is None, gives default. Default is:
|
|
|
['\\.tests$']
|
|
|
module_skip_patterns : None or sequence
|
|
|
Sequence of strings giving URIs of modules to be excluded
|
|
|
Operates on the module name including preceding URI path,
|
|
|
back to the first dot after *package_name*. For example
|
|
|
``sphinx.util.console`` results in the string to search of
|
|
|
``.util.console``
|
|
|
If is None, gives default. Default is:
|
|
|
['\\.setup$', '\\._']
|
|
|
names_from__all__ : set, optional
|
|
|
Modules listed in here will be scanned by doing ``from mod import *``,
|
|
|
rather than finding function and class definitions by scanning the
|
|
|
AST. This is intended for API modules which expose things defined in
|
|
|
other files. Modules listed here must define ``__all__`` to avoid
|
|
|
exposing everything they import.
|
|
|
'''
|
|
|
if package_skip_patterns is None:
|
|
|
package_skip_patterns = ['\\.tests$']
|
|
|
if module_skip_patterns is None:
|
|
|
module_skip_patterns = ['\\.setup$', '\\._']
|
|
|
self.package_name = package_name
|
|
|
self.rst_extension = rst_extension
|
|
|
self.package_skip_patterns = package_skip_patterns
|
|
|
self.module_skip_patterns = module_skip_patterns
|
|
|
self.names_from__all__ = names_from__all__ or set()
|
|
|
|
|
|
def get_package_name(self):
|
|
|
return self._package_name
|
|
|
|
|
|
def set_package_name(self, package_name):
|
|
|
''' Set package_name
|
|
|
|
|
|
>>> docwriter = ApiDocWriter('sphinx')
|
|
|
>>> import sphinx
|
|
|
>>> docwriter.root_path == sphinx.__path__[0]
|
|
|
True
|
|
|
>>> docwriter.package_name = 'docutils'
|
|
|
>>> import docutils
|
|
|
>>> docwriter.root_path == docutils.__path__[0]
|
|
|
True
|
|
|
'''
|
|
|
# It's also possible to imagine caching the module parsing here
|
|
|
self._package_name = package_name
|
|
|
self.root_module = import_module(package_name)
|
|
|
self.root_path = self.root_module.__path__[0]
|
|
|
self.written_modules = None
|
|
|
|
|
|
package_name = property(get_package_name, set_package_name, None,
|
|
|
'get/set package_name')
|
|
|
|
|
|
def _uri2path(self, uri):
|
|
|
''' Convert uri to absolute filepath
|
|
|
|
|
|
Parameters
|
|
|
----------
|
|
|
uri : string
|
|
|
URI of python module to return path for
|
|
|
|
|
|
Returns
|
|
|
-------
|
|
|
path : None or string
|
|
|
Returns None if there is no valid path for this URI
|
|
|
Otherwise returns absolute file system path for URI
|
|
|
|
|
|
Examples
|
|
|
--------
|
|
|
>>> docwriter = ApiDocWriter('sphinx')
|
|
|
>>> import sphinx
|
|
|
>>> modpath = sphinx.__path__[0]
|
|
|
>>> res = docwriter._uri2path('sphinx.builder')
|
|
|
>>> res == os.path.join(modpath, 'builder.py')
|
|
|
True
|
|
|
>>> res = docwriter._uri2path('sphinx')
|
|
|
>>> res == os.path.join(modpath, '__init__.py')
|
|
|
True
|
|
|
>>> docwriter._uri2path('sphinx.does_not_exist')
|
|
|
|
|
|
'''
|
|
|
if uri == self.package_name:
|
|
|
return os.path.join(self.root_path, '__init__.py')
|
|
|
path = uri.replace('.', os.path.sep)
|
|
|
path = path.replace(self.package_name + os.path.sep, '')
|
|
|
path = os.path.join(self.root_path, path)
|
|
|
# XXX maybe check for extensions as well?
|
|
|
if os.path.exists(path + '.py'): # file
|
|
|
path += '.py'
|
|
|
elif os.path.exists(os.path.join(path, '__init__.py')):
|
|
|
path = os.path.join(path, '__init__.py')
|
|
|
else:
|
|
|
return None
|
|
|
return path
|
|
|
|
|
|
def _path2uri(self, dirpath):
|
|
|
''' Convert directory path to uri '''
|
|
|
relpath = dirpath.replace(self.root_path, self.package_name)
|
|
|
if relpath.startswith(os.path.sep):
|
|
|
relpath = relpath[1:]
|
|
|
return relpath.replace(os.path.sep, '.')
|
|
|
|
|
|
def _parse_module(self, uri):
|
|
|
''' Parse module defined in *uri* '''
|
|
|
filename = self._uri2path(uri)
|
|
|
if filename is None:
|
|
|
# nothing that we could handle here.
|
|
|
return ([],[])
|
|
|
with open(filename, 'rb') as f:
|
|
|
mod = ast.parse(f.read())
|
|
|
return FuncClsScanner().scan(mod)
|
|
|
|
|
|
def _import_funcs_classes(self, uri):
|
|
|
"""Import * from uri, and separate out functions and classes."""
|
|
|
ns = {}
|
|
|
exec('from %s import *' % uri, ns)
|
|
|
funcs, classes = [], []
|
|
|
for name, obj in ns.items():
|
|
|
if inspect.isclass(obj):
|
|
|
cls = Obj(
|
|
|
name=name,
|
|
|
has_init="__init__" in obj.__dict__,
|
|
|
sphinx_options=getattr(obj, "_sphinx_options", {}),
|
|
|
)
|
|
|
classes.append(cls)
|
|
|
elif inspect.isfunction(obj):
|
|
|
funcs.append(name)
|
|
|
|
|
|
return sorted(funcs), sorted(classes, key=lambda x: x.name)
|
|
|
|
|
|
def find_funcs_classes(self, uri):
|
|
|
"""Find the functions and classes defined in the module ``uri``"""
|
|
|
if uri in self.names_from__all__:
|
|
|
# For API modules which expose things defined elsewhere, import them
|
|
|
return self._import_funcs_classes(uri)
|
|
|
else:
|
|
|
# For other modules, scan their AST to see what they define
|
|
|
return self._parse_module(uri)
|
|
|
|
|
|
def generate_api_doc(self, uri):
|
|
|
'''Make autodoc documentation template string for a module
|
|
|
|
|
|
Parameters
|
|
|
----------
|
|
|
uri : string
|
|
|
python location of module - e.g 'sphinx.builder'
|
|
|
|
|
|
Returns
|
|
|
-------
|
|
|
S : string
|
|
|
Contents of API doc
|
|
|
'''
|
|
|
# get the names of all classes and functions
|
|
|
functions, classes = self.find_funcs_classes(uri)
|
|
|
if not len(functions) and not len(classes):
|
|
|
#print ('WARNING: Empty -', uri) # dbg
|
|
|
return ''
|
|
|
|
|
|
# Make a shorter version of the uri that omits the package name for
|
|
|
# titles
|
|
|
uri_short = re.sub(r'^%s\.' % self.package_name,'',uri)
|
|
|
|
|
|
ad = '.. AUTO-GENERATED FILE -- DO NOT EDIT!\n\n'
|
|
|
|
|
|
# Set the chapter title to read 'Module:' for all modules except for the
|
|
|
# main packages
|
|
|
if '.' in uri:
|
|
|
chap_title = 'Module: :mod:`' + uri_short + '`'
|
|
|
else:
|
|
|
chap_title = ':mod:`' + uri_short + '`'
|
|
|
ad += chap_title + '\n' + self.rst_section_levels[1] * len(chap_title)
|
|
|
|
|
|
ad += '\n.. automodule:: ' + uri + '\n'
|
|
|
ad += '\n.. currentmodule:: ' + uri + '\n'
|
|
|
|
|
|
if classes:
|
|
|
subhead = str(len(classes)) + (' Classes' if len(classes) > 1 else ' Class')
|
|
|
ad += '\n'+ subhead + '\n' + \
|
|
|
self.rst_section_levels[2] * len(subhead) + '\n'
|
|
|
|
|
|
for c in classes:
|
|
|
opts = c.sphinx_options
|
|
|
ad += "\n.. autoclass:: " + c.name + "\n"
|
|
|
# must NOT exclude from index to keep cross-refs working
|
|
|
ad += " :members:\n"
|
|
|
if opts.get("show_inheritance", True):
|
|
|
ad += " :show-inheritance:\n"
|
|
|
if opts.get("show_inherited_members", False):
|
|
|
exclusions_list = opts.get("exclude_inherited_from", [])
|
|
|
exclusions = (
|
|
|
(" " + " ".join(exclusions_list)) if exclusions_list else ""
|
|
|
)
|
|
|
ad += f" :inherited-members:{exclusions}\n"
|
|
|
if c.has_init:
|
|
|
ad += '\n .. automethod:: __init__\n'
|
|
|
|
|
|
if functions:
|
|
|
subhead = str(len(functions)) + (' Functions' if len(functions) > 1 else ' Function')
|
|
|
ad += '\n'+ subhead + '\n' + \
|
|
|
self.rst_section_levels[2] * len(subhead) + '\n'
|
|
|
for f in functions:
|
|
|
# must NOT exclude from index to keep cross-refs working
|
|
|
ad += '\n.. autofunction:: ' + uri + '.' + f + '\n\n'
|
|
|
return ad
|
|
|
|
|
|
def _survives_exclude(self, matchstr, match_type):
|
|
|
''' Returns True if *matchstr* does not match patterns
|
|
|
|
|
|
``self.package_name`` removed from front of string if present
|
|
|
|
|
|
Examples
|
|
|
--------
|
|
|
>>> dw = ApiDocWriter('sphinx')
|
|
|
>>> dw._survives_exclude('sphinx.okpkg', 'package')
|
|
|
True
|
|
|
>>> dw.package_skip_patterns.append('^\\.badpkg$')
|
|
|
>>> dw._survives_exclude('sphinx.badpkg', 'package')
|
|
|
False
|
|
|
>>> dw._survives_exclude('sphinx.badpkg', 'module')
|
|
|
True
|
|
|
>>> dw._survives_exclude('sphinx.badmod', 'module')
|
|
|
True
|
|
|
>>> dw.module_skip_patterns.append('^\\.badmod$')
|
|
|
>>> dw._survives_exclude('sphinx.badmod', 'module')
|
|
|
False
|
|
|
'''
|
|
|
if match_type == 'module':
|
|
|
patterns = self.module_skip_patterns
|
|
|
elif match_type == 'package':
|
|
|
patterns = self.package_skip_patterns
|
|
|
else:
|
|
|
raise ValueError('Cannot interpret match type "%s"'
|
|
|
% match_type)
|
|
|
# Match to URI without package name
|
|
|
L = len(self.package_name)
|
|
|
if matchstr[:L] == self.package_name:
|
|
|
matchstr = matchstr[L:]
|
|
|
for pat in patterns:
|
|
|
try:
|
|
|
pat.search
|
|
|
except AttributeError:
|
|
|
pat = re.compile(pat)
|
|
|
if pat.search(matchstr):
|
|
|
return False
|
|
|
return True
|
|
|
|
|
|
def discover_modules(self):
|
|
|
''' Return module sequence discovered from ``self.package_name``
|
|
|
|
|
|
|
|
|
Parameters
|
|
|
----------
|
|
|
None
|
|
|
|
|
|
Returns
|
|
|
-------
|
|
|
mods : sequence
|
|
|
Sequence of module names within ``self.package_name``
|
|
|
|
|
|
Examples
|
|
|
--------
|
|
|
>>> dw = ApiDocWriter('sphinx')
|
|
|
>>> mods = dw.discover_modules()
|
|
|
>>> 'sphinx.util' in mods
|
|
|
True
|
|
|
>>> dw.package_skip_patterns.append('\\.util$')
|
|
|
>>> 'sphinx.util' in dw.discover_modules()
|
|
|
False
|
|
|
>>>
|
|
|
'''
|
|
|
modules = [self.package_name]
|
|
|
# raw directory parsing
|
|
|
for dirpath, dirnames, filenames in os.walk(self.root_path):
|
|
|
# Check directory names for packages
|
|
|
root_uri = self._path2uri(os.path.join(self.root_path,
|
|
|
dirpath))
|
|
|
for dirname in dirnames[:]: # copy list - we modify inplace
|
|
|
package_uri = '.'.join((root_uri, dirname))
|
|
|
if (self._uri2path(package_uri) and
|
|
|
self._survives_exclude(package_uri, 'package')):
|
|
|
modules.append(package_uri)
|
|
|
else:
|
|
|
dirnames.remove(dirname)
|
|
|
# Check filenames for modules
|
|
|
for filename in filenames:
|
|
|
module_name = filename[:-3]
|
|
|
module_uri = '.'.join((root_uri, module_name))
|
|
|
if (self._uri2path(module_uri) and
|
|
|
self._survives_exclude(module_uri, 'module')):
|
|
|
modules.append(module_uri)
|
|
|
return sorted(modules)
|
|
|
|
|
|
def write_modules_api(self, modules,outdir):
|
|
|
# write the list
|
|
|
written_modules = []
|
|
|
for m in modules:
|
|
|
api_str = self.generate_api_doc(m)
|
|
|
if not api_str:
|
|
|
continue
|
|
|
# write out to file
|
|
|
outfile = os.path.join(outdir, m + self.rst_extension)
|
|
|
with open(outfile, "wt", encoding="utf-8") as fileobj:
|
|
|
fileobj.write(api_str)
|
|
|
written_modules.append(m)
|
|
|
self.written_modules = written_modules
|
|
|
|
|
|
def write_api_docs(self, outdir):
|
|
|
"""Generate API reST files.
|
|
|
|
|
|
Parameters
|
|
|
----------
|
|
|
outdir : string
|
|
|
Directory name in which to store files
|
|
|
We create automatic filenames for each module
|
|
|
|
|
|
Returns
|
|
|
-------
|
|
|
None
|
|
|
|
|
|
Notes
|
|
|
-----
|
|
|
Sets self.written_modules to list of written modules
|
|
|
"""
|
|
|
if not os.path.exists(outdir):
|
|
|
os.mkdir(outdir)
|
|
|
# compose list of modules
|
|
|
modules = self.discover_modules()
|
|
|
self.write_modules_api(modules,outdir)
|
|
|
|
|
|
def write_index(self, outdir, path='gen.rst', relative_to=None):
|
|
|
"""Make a reST API index file from written files
|
|
|
|
|
|
Parameters
|
|
|
----------
|
|
|
outdir : string
|
|
|
Directory to which to write generated index file
|
|
|
path : string
|
|
|
Filename to write index to
|
|
|
relative_to : string
|
|
|
path to which written filenames are relative. This
|
|
|
component of the written file path will be removed from
|
|
|
outdir, in the generated index. Default is None, meaning,
|
|
|
leave path as it is.
|
|
|
"""
|
|
|
if self.written_modules is None:
|
|
|
raise ValueError('No modules written')
|
|
|
# Get full filename path
|
|
|
path = os.path.join(outdir, path)
|
|
|
# Path written into index is relative to rootpath
|
|
|
if relative_to is not None:
|
|
|
relpath = outdir.replace(relative_to + os.path.sep, '')
|
|
|
else:
|
|
|
relpath = outdir
|
|
|
with open(path, "wt", encoding="utf-8") as idx:
|
|
|
w = idx.write
|
|
|
w('.. AUTO-GENERATED FILE -- DO NOT EDIT!\n\n')
|
|
|
w('.. autosummary::\n'
|
|
|
' :toctree: %s\n\n' % relpath)
|
|
|
for mod in self.written_modules:
|
|
|
w(' %s\n' % mod)
|
|
|
|
