"""Attempt to generate templates for module reference with Sphinx XXX - we exclude extension modules To include extension modules, first identify them as valid in the ``_uri2path`` method, then handle them in the ``_parse_module`` script. We get functions and classes by parsing the text of .py files. Alternatively we could import the modules for discovery, and we'd have to do that for extension modules. This would involve changing the ``_parse_module`` method to work via import and introspection, and might involve changing ``discover_modules`` (which determines which files are modules, and therefore which module URIs will be passed to ``_parse_module``). NOTE: this is a modified version of a script originally shipped with the PyMVPA project, which we've adapted for NIPY use. PyMVPA is an MIT-licensed project.""" # Stdlib imports import ast import os import re class Obj(object): '''Namespace to hold arbitrary information.''' def __init__(self, **kwargs): for k, v in kwargs.items(): setattr(self, k, v) class FuncClsScanner(ast.NodeVisitor): """Scan a module for top-level functions and classes. Skips objects with an @undoc decorator, or a name starting with '_'. """ def __init__(self): ast.NodeVisitor.__init__(self) self.classes = [] self.classes_seen = set() self.functions = [] @staticmethod def has_undoc_decorator(node): return any(isinstance(d, ast.Name) and d.id == 'undoc' \ for d in node.decorator_list) def visit_FunctionDef(self, node): if not (node.name.startswith('_') or self.has_undoc_decorator(node)) \ and node.name not in self.functions: self.functions.append(node.name) def visit_ClassDef(self, node): if not (node.name.startswith('_') or self.has_undoc_decorator(node)) \ and node.name not in self.classes_seen: cls = Obj(name=node.name) cls.has_init = any(isinstance(n, ast.FunctionDef) and \ n.name=='__init__' for n in node.body) self.classes.append(cls) self.classes_seen.add(node.name) def scan(self, mod): self.visit(mod) return self.functions, self.classes # Functions and classes class ApiDocWriter(object): ''' Class for automatic detection and parsing of API docs to Sphinx-parsable reST format''' # only separating first two levels rst_section_levels = ['*', '=', '-', '~', '^'] def __init__(self, package_name, rst_extension='.rst', package_skip_patterns=None, module_skip_patterns=None, ): ''' Initialize package for parsing Parameters ---------- package_name : string Name of the top-level package. *package_name* must be the name of an importable package rst_extension : string, optional Extension for reST files, default '.rst' package_skip_patterns : None or sequence of {strings, regexps} Sequence of strings giving URIs of packages to be excluded Operates on the package path, starting at (including) the first dot in the package path, after *package_name* - so, if *package_name* is ``sphinx``, then ``sphinx.util`` will result in ``.util`` being passed for earching by these regexps. If is None, gives default. Default is: ['\.tests$'] module_skip_patterns : None or sequence Sequence of strings giving URIs of modules to be excluded Operates on the module name including preceding URI path, back to the first dot after *package_name*. For example ``sphinx.util.console`` results in the string to search of ``.util.console`` If is None, gives default. Default is: ['\.setup$', '\._'] ''' if package_skip_patterns is None: package_skip_patterns = ['\\.tests$'] if module_skip_patterns is None: module_skip_patterns = ['\\.setup$', '\\._'] self.package_name = package_name self.rst_extension = rst_extension self.package_skip_patterns = package_skip_patterns self.module_skip_patterns = module_skip_patterns def get_package_name(self): return self._package_name def set_package_name(self, package_name): ''' Set package_name >>> docwriter = ApiDocWriter('sphinx') >>> import sphinx >>> docwriter.root_path == sphinx.__path__[0] True >>> docwriter.package_name = 'docutils' >>> import docutils >>> docwriter.root_path == docutils.__path__[0] True ''' # It's also possible to imagine caching the module parsing here self._package_name = package_name self.root_module = __import__(package_name) self.root_path = self.root_module.__path__[0] self.written_modules = None package_name = property(get_package_name, set_package_name, None, 'get/set package_name') def _uri2path(self, uri): ''' Convert uri to absolute filepath Parameters ---------- uri : string URI of python module to return path for Returns ------- path : None or string Returns None if there is no valid path for this URI Otherwise returns absolute file system path for URI Examples -------- >>> docwriter = ApiDocWriter('sphinx') >>> import sphinx >>> modpath = sphinx.__path__[0] >>> res = docwriter._uri2path('sphinx.builder') >>> res == os.path.join(modpath, 'builder.py') True >>> res = docwriter._uri2path('sphinx') >>> res == os.path.join(modpath, '__init__.py') True >>> docwriter._uri2path('sphinx.does_not_exist') ''' if uri == self.package_name: return os.path.join(self.root_path, '__init__.py') path = uri.replace('.', os.path.sep) path = path.replace(self.package_name + os.path.sep, '') path = os.path.join(self.root_path, path) # XXX maybe check for extensions as well? if os.path.exists(path + '.py'): # file path += '.py' elif os.path.exists(os.path.join(path, '__init__.py')): path = os.path.join(path, '__init__.py') else: return None return path def _path2uri(self, dirpath): ''' Convert directory path to uri ''' relpath = dirpath.replace(self.root_path, self.package_name) if relpath.startswith(os.path.sep): relpath = relpath[1:] return relpath.replace(os.path.sep, '.') def _parse_module(self, uri): ''' Parse module defined in *uri* ''' filename = self._uri2path(uri) if filename is None: # nothing that we could handle here. return ([],[]) with open(filename, 'rb') as f: mod = ast.parse(f.read()) return FuncClsScanner().scan(mod) def generate_api_doc(self, uri): '''Make autodoc documentation template string for a module Parameters ---------- uri : string python location of module - e.g 'sphinx.builder' Returns ------- S : string Contents of API doc ''' # get the names of all classes and functions functions, classes = self._parse_module(uri) if not len(functions) and not len(classes): print 'WARNING: Empty -',uri # dbg return '' # Make a shorter version of the uri that omits the package name for # titles uri_short = re.sub(r'^%s\.' % self.package_name,'',uri) ad = '.. AUTO-GENERATED FILE -- DO NOT EDIT!\n\n' # Set the chapter title to read 'Module:' for all modules except for the # main packages if '.' in uri: chap_title = 'Module: :mod:`' + uri_short + '`' else: chap_title = ':mod:`' + uri_short + '`' ad += chap_title + '\n' + self.rst_section_levels[1] * len(chap_title) ad += '\n.. automodule:: ' + uri + '\n' ad += '\n.. currentmodule:: ' + uri + '\n' if classes: subhead = str(len(classes)) + (' Classes' if len(classes) > 1 else ' Class') ad += '\n'+ subhead + '\n' + \ self.rst_section_levels[2] * len(subhead) + '\n' for c in classes: ad += '\n.. autoclass:: ' + c.name + '\n' # must NOT exclude from index to keep cross-refs working ad += ' :members:\n' \ ' :show-inheritance:\n' if c.has_init: ad += '\n .. automethod:: __init__\n' if functions: subhead = str(len(functions)) + (' Functions' if len(functions) > 1 else ' Function') ad += '\n'+ subhead + '\n' + \ self.rst_section_levels[2] * len(subhead) + '\n' for f in functions: # must NOT exclude from index to keep cross-refs working ad += '\n.. autofunction:: ' + uri + '.' + f + '\n\n' return ad def _survives_exclude(self, matchstr, match_type): ''' Returns True if *matchstr* does not match patterns ``self.package_name`` removed from front of string if present Examples -------- >>> dw = ApiDocWriter('sphinx') >>> dw._survives_exclude('sphinx.okpkg', 'package') True >>> dw.package_skip_patterns.append('^\\.badpkg$') >>> dw._survives_exclude('sphinx.badpkg', 'package') False >>> dw._survives_exclude('sphinx.badpkg', 'module') True >>> dw._survives_exclude('sphinx.badmod', 'module') True >>> dw.module_skip_patterns.append('^\\.badmod$') >>> dw._survives_exclude('sphinx.badmod', 'module') False ''' if match_type == 'module': patterns = self.module_skip_patterns elif match_type == 'package': patterns = self.package_skip_patterns else: raise ValueError('Cannot interpret match type "%s"' % match_type) # Match to URI without package name L = len(self.package_name) if matchstr[:L] == self.package_name: matchstr = matchstr[L:] for pat in patterns: try: pat.search except AttributeError: pat = re.compile(pat) if pat.search(matchstr): return False return True def discover_modules(self): ''' Return module sequence discovered from ``self.package_name`` Parameters ---------- None Returns ------- mods : sequence Sequence of module names within ``self.package_name`` Examples -------- >>> dw = ApiDocWriter('sphinx') >>> mods = dw.discover_modules() >>> 'sphinx.util' in mods True >>> dw.package_skip_patterns.append('\.util$') >>> 'sphinx.util' in dw.discover_modules() False >>> ''' modules = [self.package_name] # raw directory parsing for dirpath, dirnames, filenames in os.walk(self.root_path): # Check directory names for packages root_uri = self._path2uri(os.path.join(self.root_path, dirpath)) for dirname in dirnames[:]: # copy list - we modify inplace package_uri = '.'.join((root_uri, dirname)) if (self._uri2path(package_uri) and self._survives_exclude(package_uri, 'package')): modules.append(package_uri) else: dirnames.remove(dirname) # Check filenames for modules for filename in filenames: module_name = filename[:-3] module_uri = '.'.join((root_uri, module_name)) if (self._uri2path(module_uri) and self._survives_exclude(module_uri, 'module')): modules.append(module_uri) return sorted(modules) def write_modules_api(self, modules,outdir): # write the list written_modules = [] for m in modules: api_str = self.generate_api_doc(m) if not api_str: continue # write out to file outfile = os.path.join(outdir, m + self.rst_extension) fileobj = open(outfile, 'wt') fileobj.write(api_str) fileobj.close() written_modules.append(m) self.written_modules = written_modules def write_api_docs(self, outdir): """Generate API reST files. Parameters ---------- outdir : string Directory name in which to store files We create automatic filenames for each module Returns ------- None Notes ----- Sets self.written_modules to list of written modules """ if not os.path.exists(outdir): os.mkdir(outdir) # compose list of modules modules = self.discover_modules() self.write_modules_api(modules,outdir) def write_index(self, outdir, froot='gen', relative_to=None): """Make a reST API index file from written files Parameters ---------- path : string Filename to write index to outdir : string Directory to which to write generated index file froot : string, optional root (filename without extension) of filename to write to Defaults to 'gen'. We add ``self.rst_extension``. relative_to : string path to which written filenames are relative. This component of the written file path will be removed from outdir, in the generated index. Default is None, meaning, leave path as it is. """ if self.written_modules is None: raise ValueError('No modules written') # Get full filename path path = os.path.join(outdir, froot+self.rst_extension) # Path written into index is relative to rootpath if relative_to is not None: relpath = outdir.replace(relative_to + os.path.sep, '') else: relpath = outdir idx = open(path,'wt') w = idx.write w('.. AUTO-GENERATED FILE -- DO NOT EDIT!\n\n') w('.. toctree::\n\n') for f in self.written_modules: w(' %s\n' % os.path.join(relpath,f)) idx.close()