##// END OF EJS Templates
Don't document classes defined in if __name__ == '__main__' blocks
Thomas Kluyver -
Show More
@@ -1,413 +1,421 b''
1 1 """Attempt to generate templates for module reference with Sphinx
2 2
3 3 XXX - we exclude extension modules
4 4
5 5 To include extension modules, first identify them as valid in the
6 6 ``_uri2path`` method, then handle them in the ``_parse_module`` script.
7 7
8 8 We get functions and classes by parsing the text of .py files.
9 9 Alternatively we could import the modules for discovery, and we'd have
10 10 to do that for extension modules. This would involve changing the
11 11 ``_parse_module`` method to work via import and introspection, and
12 12 might involve changing ``discover_modules`` (which determines which
13 13 files are modules, and therefore which module URIs will be passed to
14 14 ``_parse_module``).
15 15
16 16 NOTE: this is a modified version of a script originally shipped with the
17 17 PyMVPA project, which we've adapted for NIPY use. PyMVPA is an MIT-licensed
18 18 project."""
19 19
20 20 from __future__ import print_function
21 21
22 22 # Stdlib imports
23 23 import ast
24 24 import os
25 25 import re
26 26
27 27 class Obj(object):
28 28 '''Namespace to hold arbitrary information.'''
29 29 def __init__(self, **kwargs):
30 30 for k, v in kwargs.items():
31 31 setattr(self, k, v)
32 32
33 33 class FuncClsScanner(ast.NodeVisitor):
34 34 """Scan a module for top-level functions and classes.
35 35
36 36 Skips objects with an @undoc decorator, or a name starting with '_'.
37 37 """
38 38 def __init__(self):
39 39 ast.NodeVisitor.__init__(self)
40 40 self.classes = []
41 41 self.classes_seen = set()
42 42 self.functions = []
43 43
44 44 @staticmethod
45 45 def has_undoc_decorator(node):
46 46 return any(isinstance(d, ast.Name) and d.id == 'undoc' \
47 47 for d in node.decorator_list)
48
49 def visit_If(self, node):
50 if isinstance(node.test, ast.Compare) \
51 and isinstance(node.test.left, ast.Name) \
52 and node.test.left.id == '__name__':
53 return # Ignore classes defined in "if __name__ == '__main__':"
54
55 self.generic_visit(node)
48 56
49 57 def visit_FunctionDef(self, node):
50 58 if not (node.name.startswith('_') or self.has_undoc_decorator(node)) \
51 59 and node.name not in self.functions:
52 60 self.functions.append(node.name)
53 61
54 62 def visit_ClassDef(self, node):
55 63 if not (node.name.startswith('_') or self.has_undoc_decorator(node)) \
56 64 and node.name not in self.classes_seen:
57 65 cls = Obj(name=node.name)
58 66 cls.has_init = any(isinstance(n, ast.FunctionDef) and \
59 67 n.name=='__init__' for n in node.body)
60 68 self.classes.append(cls)
61 69 self.classes_seen.add(node.name)
62 70
63 71 def scan(self, mod):
64 72 self.visit(mod)
65 73 return self.functions, self.classes
66 74
67 75 # Functions and classes
68 76 class ApiDocWriter(object):
69 77 ''' Class for automatic detection and parsing of API docs
70 78 to Sphinx-parsable reST format'''
71 79
72 80 # only separating first two levels
73 81 rst_section_levels = ['*', '=', '-', '~', '^']
74 82
75 83 def __init__(self,
76 84 package_name,
77 85 rst_extension='.rst',
78 86 package_skip_patterns=None,
79 87 module_skip_patterns=None,
80 88 ):
81 89 ''' Initialize package for parsing
82 90
83 91 Parameters
84 92 ----------
85 93 package_name : string
86 94 Name of the top-level package. *package_name* must be the
87 95 name of an importable package
88 96 rst_extension : string, optional
89 97 Extension for reST files, default '.rst'
90 98 package_skip_patterns : None or sequence of {strings, regexps}
91 99 Sequence of strings giving URIs of packages to be excluded
92 100 Operates on the package path, starting at (including) the
93 101 first dot in the package path, after *package_name* - so,
94 102 if *package_name* is ``sphinx``, then ``sphinx.util`` will
95 103 result in ``.util`` being passed for earching by these
96 104 regexps. If is None, gives default. Default is:
97 105 ['\.tests$']
98 106 module_skip_patterns : None or sequence
99 107 Sequence of strings giving URIs of modules to be excluded
100 108 Operates on the module name including preceding URI path,
101 109 back to the first dot after *package_name*. For example
102 110 ``sphinx.util.console`` results in the string to search of
103 111 ``.util.console``
104 112 If is None, gives default. Default is:
105 113 ['\.setup$', '\._']
106 114 '''
107 115 if package_skip_patterns is None:
108 116 package_skip_patterns = ['\\.tests$']
109 117 if module_skip_patterns is None:
110 118 module_skip_patterns = ['\\.setup$', '\\._']
111 119 self.package_name = package_name
112 120 self.rst_extension = rst_extension
113 121 self.package_skip_patterns = package_skip_patterns
114 122 self.module_skip_patterns = module_skip_patterns
115 123
116 124 def get_package_name(self):
117 125 return self._package_name
118 126
119 127 def set_package_name(self, package_name):
120 128 ''' Set package_name
121 129
122 130 >>> docwriter = ApiDocWriter('sphinx')
123 131 >>> import sphinx
124 132 >>> docwriter.root_path == sphinx.__path__[0]
125 133 True
126 134 >>> docwriter.package_name = 'docutils'
127 135 >>> import docutils
128 136 >>> docwriter.root_path == docutils.__path__[0]
129 137 True
130 138 '''
131 139 # It's also possible to imagine caching the module parsing here
132 140 self._package_name = package_name
133 141 self.root_module = __import__(package_name)
134 142 self.root_path = self.root_module.__path__[0]
135 143 self.written_modules = None
136 144
137 145 package_name = property(get_package_name, set_package_name, None,
138 146 'get/set package_name')
139 147
140 148 def _uri2path(self, uri):
141 149 ''' Convert uri to absolute filepath
142 150
143 151 Parameters
144 152 ----------
145 153 uri : string
146 154 URI of python module to return path for
147 155
148 156 Returns
149 157 -------
150 158 path : None or string
151 159 Returns None if there is no valid path for this URI
152 160 Otherwise returns absolute file system path for URI
153 161
154 162 Examples
155 163 --------
156 164 >>> docwriter = ApiDocWriter('sphinx')
157 165 >>> import sphinx
158 166 >>> modpath = sphinx.__path__[0]
159 167 >>> res = docwriter._uri2path('sphinx.builder')
160 168 >>> res == os.path.join(modpath, 'builder.py')
161 169 True
162 170 >>> res = docwriter._uri2path('sphinx')
163 171 >>> res == os.path.join(modpath, '__init__.py')
164 172 True
165 173 >>> docwriter._uri2path('sphinx.does_not_exist')
166 174
167 175 '''
168 176 if uri == self.package_name:
169 177 return os.path.join(self.root_path, '__init__.py')
170 178 path = uri.replace('.', os.path.sep)
171 179 path = path.replace(self.package_name + os.path.sep, '')
172 180 path = os.path.join(self.root_path, path)
173 181 # XXX maybe check for extensions as well?
174 182 if os.path.exists(path + '.py'): # file
175 183 path += '.py'
176 184 elif os.path.exists(os.path.join(path, '__init__.py')):
177 185 path = os.path.join(path, '__init__.py')
178 186 else:
179 187 return None
180 188 return path
181 189
182 190 def _path2uri(self, dirpath):
183 191 ''' Convert directory path to uri '''
184 192 relpath = dirpath.replace(self.root_path, self.package_name)
185 193 if relpath.startswith(os.path.sep):
186 194 relpath = relpath[1:]
187 195 return relpath.replace(os.path.sep, '.')
188 196
189 197 def _parse_module(self, uri):
190 198 ''' Parse module defined in *uri* '''
191 199 filename = self._uri2path(uri)
192 200 if filename is None:
193 201 # nothing that we could handle here.
194 202 return ([],[])
195 203 with open(filename, 'rb') as f:
196 204 mod = ast.parse(f.read())
197 205 return FuncClsScanner().scan(mod)
198 206
199 207 def generate_api_doc(self, uri):
200 208 '''Make autodoc documentation template string for a module
201 209
202 210 Parameters
203 211 ----------
204 212 uri : string
205 213 python location of module - e.g 'sphinx.builder'
206 214
207 215 Returns
208 216 -------
209 217 S : string
210 218 Contents of API doc
211 219 '''
212 220 # get the names of all classes and functions
213 221 functions, classes = self._parse_module(uri)
214 222 if not len(functions) and not len(classes):
215 223 print ('WARNING: Empty -', uri) # dbg
216 224 return ''
217 225
218 226 # Make a shorter version of the uri that omits the package name for
219 227 # titles
220 228 uri_short = re.sub(r'^%s\.' % self.package_name,'',uri)
221 229
222 230 ad = '.. AUTO-GENERATED FILE -- DO NOT EDIT!\n\n'
223 231
224 232 # Set the chapter title to read 'Module:' for all modules except for the
225 233 # main packages
226 234 if '.' in uri:
227 235 chap_title = 'Module: :mod:`' + uri_short + '`'
228 236 else:
229 237 chap_title = ':mod:`' + uri_short + '`'
230 238 ad += chap_title + '\n' + self.rst_section_levels[1] * len(chap_title)
231 239
232 240 ad += '\n.. automodule:: ' + uri + '\n'
233 241 ad += '\n.. currentmodule:: ' + uri + '\n'
234 242
235 243 if classes:
236 244 subhead = str(len(classes)) + (' Classes' if len(classes) > 1 else ' Class')
237 245 ad += '\n'+ subhead + '\n' + \
238 246 self.rst_section_levels[2] * len(subhead) + '\n'
239 247
240 248 for c in classes:
241 249 ad += '\n.. autoclass:: ' + c.name + '\n'
242 250 # must NOT exclude from index to keep cross-refs working
243 251 ad += ' :members:\n' \
244 252 ' :show-inheritance:\n'
245 253 if c.has_init:
246 254 ad += '\n .. automethod:: __init__\n'
247 255
248 256 if functions:
249 257 subhead = str(len(functions)) + (' Functions' if len(functions) > 1 else ' Function')
250 258 ad += '\n'+ subhead + '\n' + \
251 259 self.rst_section_levels[2] * len(subhead) + '\n'
252 260 for f in functions:
253 261 # must NOT exclude from index to keep cross-refs working
254 262 ad += '\n.. autofunction:: ' + uri + '.' + f + '\n\n'
255 263 return ad
256 264
257 265 def _survives_exclude(self, matchstr, match_type):
258 266 ''' Returns True if *matchstr* does not match patterns
259 267
260 268 ``self.package_name`` removed from front of string if present
261 269
262 270 Examples
263 271 --------
264 272 >>> dw = ApiDocWriter('sphinx')
265 273 >>> dw._survives_exclude('sphinx.okpkg', 'package')
266 274 True
267 275 >>> dw.package_skip_patterns.append('^\\.badpkg$')
268 276 >>> dw._survives_exclude('sphinx.badpkg', 'package')
269 277 False
270 278 >>> dw._survives_exclude('sphinx.badpkg', 'module')
271 279 True
272 280 >>> dw._survives_exclude('sphinx.badmod', 'module')
273 281 True
274 282 >>> dw.module_skip_patterns.append('^\\.badmod$')
275 283 >>> dw._survives_exclude('sphinx.badmod', 'module')
276 284 False
277 285 '''
278 286 if match_type == 'module':
279 287 patterns = self.module_skip_patterns
280 288 elif match_type == 'package':
281 289 patterns = self.package_skip_patterns
282 290 else:
283 291 raise ValueError('Cannot interpret match type "%s"'
284 292 % match_type)
285 293 # Match to URI without package name
286 294 L = len(self.package_name)
287 295 if matchstr[:L] == self.package_name:
288 296 matchstr = matchstr[L:]
289 297 for pat in patterns:
290 298 try:
291 299 pat.search
292 300 except AttributeError:
293 301 pat = re.compile(pat)
294 302 if pat.search(matchstr):
295 303 return False
296 304 return True
297 305
298 306 def discover_modules(self):
299 307 ''' Return module sequence discovered from ``self.package_name``
300 308
301 309
302 310 Parameters
303 311 ----------
304 312 None
305 313
306 314 Returns
307 315 -------
308 316 mods : sequence
309 317 Sequence of module names within ``self.package_name``
310 318
311 319 Examples
312 320 --------
313 321 >>> dw = ApiDocWriter('sphinx')
314 322 >>> mods = dw.discover_modules()
315 323 >>> 'sphinx.util' in mods
316 324 True
317 325 >>> dw.package_skip_patterns.append('\.util$')
318 326 >>> 'sphinx.util' in dw.discover_modules()
319 327 False
320 328 >>>
321 329 '''
322 330 modules = [self.package_name]
323 331 # raw directory parsing
324 332 for dirpath, dirnames, filenames in os.walk(self.root_path):
325 333 # Check directory names for packages
326 334 root_uri = self._path2uri(os.path.join(self.root_path,
327 335 dirpath))
328 336 for dirname in dirnames[:]: # copy list - we modify inplace
329 337 package_uri = '.'.join((root_uri, dirname))
330 338 if (self._uri2path(package_uri) and
331 339 self._survives_exclude(package_uri, 'package')):
332 340 modules.append(package_uri)
333 341 else:
334 342 dirnames.remove(dirname)
335 343 # Check filenames for modules
336 344 for filename in filenames:
337 345 module_name = filename[:-3]
338 346 module_uri = '.'.join((root_uri, module_name))
339 347 if (self._uri2path(module_uri) and
340 348 self._survives_exclude(module_uri, 'module')):
341 349 modules.append(module_uri)
342 350 return sorted(modules)
343 351
344 352 def write_modules_api(self, modules,outdir):
345 353 # write the list
346 354 written_modules = []
347 355 for m in modules:
348 356 api_str = self.generate_api_doc(m)
349 357 if not api_str:
350 358 continue
351 359 # write out to file
352 360 outfile = os.path.join(outdir,
353 361 m + self.rst_extension)
354 362 fileobj = open(outfile, 'wt')
355 363 fileobj.write(api_str)
356 364 fileobj.close()
357 365 written_modules.append(m)
358 366 self.written_modules = written_modules
359 367
360 368 def write_api_docs(self, outdir):
361 369 """Generate API reST files.
362 370
363 371 Parameters
364 372 ----------
365 373 outdir : string
366 374 Directory name in which to store files
367 375 We create automatic filenames for each module
368 376
369 377 Returns
370 378 -------
371 379 None
372 380
373 381 Notes
374 382 -----
375 383 Sets self.written_modules to list of written modules
376 384 """
377 385 if not os.path.exists(outdir):
378 386 os.mkdir(outdir)
379 387 # compose list of modules
380 388 modules = self.discover_modules()
381 389 self.write_modules_api(modules,outdir)
382 390
383 391 def write_index(self, outdir, path='gen.rst', relative_to=None):
384 392 """Make a reST API index file from written files
385 393
386 394 Parameters
387 395 ----------
388 396 outdir : string
389 397 Directory to which to write generated index file
390 398 path : string
391 399 Filename to write index to
392 400 relative_to : string
393 401 path to which written filenames are relative. This
394 402 component of the written file path will be removed from
395 403 outdir, in the generated index. Default is None, meaning,
396 404 leave path as it is.
397 405 """
398 406 if self.written_modules is None:
399 407 raise ValueError('No modules written')
400 408 # Get full filename path
401 409 path = os.path.join(outdir, path)
402 410 # Path written into index is relative to rootpath
403 411 if relative_to is not None:
404 412 relpath = outdir.replace(relative_to + os.path.sep, '')
405 413 else:
406 414 relpath = outdir
407 415 idx = open(path,'wt')
408 416 w = idx.write
409 417 w('.. AUTO-GENERATED FILE -- DO NOT EDIT!\n\n')
410 418 w('.. toctree::\n\n')
411 419 for f in self.written_modules:
412 420 w(' %s\n' % os.path.join(relpath,f))
413 421 idx.close()
General Comments 0
You need to be logged in to leave comments. Login now