Show More
@@ -1,406 +1,414 b'' | |||
|
1 | 1 | """Attempt to generate templates for module reference with Sphinx |
|
2 | 2 | |
|
3 | 3 | XXX - we exclude extension modules |
|
4 | 4 | |
|
5 | 5 | To include extension modules, first identify them as valid in the |
|
6 | 6 | ``_uri2path`` method, then handle them in the ``_parse_module`` script. |
|
7 | 7 | |
|
8 | 8 | We get functions and classes by parsing the text of .py files. |
|
9 | 9 | Alternatively we could import the modules for discovery, and we'd have |
|
10 | 10 | to do that for extension modules. This would involve changing the |
|
11 | 11 | ``_parse_module`` method to work via import and introspection, and |
|
12 | 12 | might involve changing ``discover_modules`` (which determines which |
|
13 | 13 | files are modules, and therefore which module URIs will be passed to |
|
14 | 14 | ``_parse_module``). |
|
15 | 15 | |
|
16 | 16 | NOTE: this is a modified version of a script originally shipped with the |
|
17 | 17 | PyMVPA project, which we've adapted for NIPY use. PyMVPA is an MIT-licensed |
|
18 | 18 | project.""" |
|
19 | 19 | |
|
20 | 20 | # Stdlib imports |
|
21 | 21 | import ast |
|
22 | 22 | import os |
|
23 | 23 | import re |
|
24 | 24 | |
|
25 | 25 | class Obj(object): |
|
26 | 26 | '''Namespace to hold arbitrary information.''' |
|
27 | 27 | def __init__(self, **kwargs): |
|
28 | 28 | for k, v in kwargs.items(): |
|
29 | 29 | setattr(self, k, v) |
|
30 | ||
|
31 | class FuncClsScanner(ast.NodeVisitor): | |
|
32 | """Scan a module for top-level functions and classes. | |
|
33 | ||
|
34 | Skips objects with an @undoc decorator, or a name starting with '_'. | |
|
35 | """ | |
|
36 | def __init__(self): | |
|
37 | ast.NodeVisitor.__init__(self) | |
|
38 | self.classes = [] | |
|
39 | self.classes_seen = set() | |
|
40 | self.functions = [] | |
|
41 | ||
|
42 | @staticmethod | |
|
43 | def has_undoc_decorator(node): | |
|
44 | return any(isinstance(d, ast.Name) and d.id == 'undoc' \ | |
|
45 | for d in node.decorator_list) | |
|
46 | ||
|
47 | def visit_FunctionDef(self, node): | |
|
48 | if not (node.name.startswith('_') or self.has_undoc_decorator(node)) \ | |
|
49 | and node.name not in self.functions: | |
|
50 | self.functions.append(node.name) | |
|
51 | ||
|
52 | def visit_ClassDef(self, node): | |
|
53 | if not (node.name.startswith('_') or self.has_undoc_decorator(node)) \ | |
|
54 | and node.name not in self.classes_seen: | |
|
55 | cls = Obj(name=node.name) | |
|
56 | cls.has_init = any(isinstance(n, ast.FunctionDef) and \ | |
|
57 | n.name=='__init__' for n in node.body) | |
|
58 | self.classes.append(cls) | |
|
59 | self.classes_seen.add(node.name) | |
|
60 | ||
|
61 | def scan(self, mod): | |
|
62 | self.visit(mod) | |
|
63 | return self.functions, self.classes | |
|
30 | 64 | |
|
31 | 65 | # Functions and classes |
|
32 | 66 | class ApiDocWriter(object): |
|
33 | 67 | ''' Class for automatic detection and parsing of API docs |
|
34 | 68 | to Sphinx-parsable reST format''' |
|
35 | 69 | |
|
36 | 70 | # only separating first two levels |
|
37 | 71 | rst_section_levels = ['*', '=', '-', '~', '^'] |
|
38 | 72 | |
|
39 | 73 | def __init__(self, |
|
40 | 74 | package_name, |
|
41 | 75 | rst_extension='.rst', |
|
42 | 76 | package_skip_patterns=None, |
|
43 | 77 | module_skip_patterns=None, |
|
44 | 78 | ): |
|
45 | 79 | ''' Initialize package for parsing |
|
46 | 80 | |
|
47 | 81 | Parameters |
|
48 | 82 | ---------- |
|
49 | 83 | package_name : string |
|
50 | 84 | Name of the top-level package. *package_name* must be the |
|
51 | 85 | name of an importable package |
|
52 | 86 | rst_extension : string, optional |
|
53 | 87 | Extension for reST files, default '.rst' |
|
54 | 88 | package_skip_patterns : None or sequence of {strings, regexps} |
|
55 | 89 | Sequence of strings giving URIs of packages to be excluded |
|
56 | 90 | Operates on the package path, starting at (including) the |
|
57 | 91 | first dot in the package path, after *package_name* - so, |
|
58 | 92 | if *package_name* is ``sphinx``, then ``sphinx.util`` will |
|
59 | 93 | result in ``.util`` being passed for earching by these |
|
60 | 94 | regexps. If is None, gives default. Default is: |
|
61 | 95 | ['\.tests$'] |
|
62 | 96 | module_skip_patterns : None or sequence |
|
63 | 97 | Sequence of strings giving URIs of modules to be excluded |
|
64 | 98 | Operates on the module name including preceding URI path, |
|
65 | 99 | back to the first dot after *package_name*. For example |
|
66 | 100 | ``sphinx.util.console`` results in the string to search of |
|
67 | 101 | ``.util.console`` |
|
68 | 102 | If is None, gives default. Default is: |
|
69 | 103 | ['\.setup$', '\._'] |
|
70 | 104 | ''' |
|
71 | 105 | if package_skip_patterns is None: |
|
72 | 106 | package_skip_patterns = ['\\.tests$'] |
|
73 | 107 | if module_skip_patterns is None: |
|
74 | 108 | module_skip_patterns = ['\\.setup$', '\\._'] |
|
75 | 109 | self.package_name = package_name |
|
76 | 110 | self.rst_extension = rst_extension |
|
77 | 111 | self.package_skip_patterns = package_skip_patterns |
|
78 | 112 | self.module_skip_patterns = module_skip_patterns |
|
79 | 113 | |
|
80 | 114 | def get_package_name(self): |
|
81 | 115 | return self._package_name |
|
82 | 116 | |
|
83 | 117 | def set_package_name(self, package_name): |
|
84 | 118 | ''' Set package_name |
|
85 | 119 | |
|
86 | 120 | >>> docwriter = ApiDocWriter('sphinx') |
|
87 | 121 | >>> import sphinx |
|
88 | 122 | >>> docwriter.root_path == sphinx.__path__[0] |
|
89 | 123 | True |
|
90 | 124 | >>> docwriter.package_name = 'docutils' |
|
91 | 125 | >>> import docutils |
|
92 | 126 | >>> docwriter.root_path == docutils.__path__[0] |
|
93 | 127 | True |
|
94 | 128 | ''' |
|
95 | 129 | # It's also possible to imagine caching the module parsing here |
|
96 | 130 | self._package_name = package_name |
|
97 | 131 | self.root_module = __import__(package_name) |
|
98 | 132 | self.root_path = self.root_module.__path__[0] |
|
99 | 133 | self.written_modules = None |
|
100 | 134 | |
|
101 | 135 | package_name = property(get_package_name, set_package_name, None, |
|
102 | 136 | 'get/set package_name') |
|
103 | 137 | |
|
104 | 138 | def _uri2path(self, uri): |
|
105 | 139 | ''' Convert uri to absolute filepath |
|
106 | 140 | |
|
107 | 141 | Parameters |
|
108 | 142 | ---------- |
|
109 | 143 | uri : string |
|
110 | 144 | URI of python module to return path for |
|
111 | 145 | |
|
112 | 146 | Returns |
|
113 | 147 | ------- |
|
114 | 148 | path : None or string |
|
115 | 149 | Returns None if there is no valid path for this URI |
|
116 | 150 | Otherwise returns absolute file system path for URI |
|
117 | 151 | |
|
118 | 152 | Examples |
|
119 | 153 | -------- |
|
120 | 154 | >>> docwriter = ApiDocWriter('sphinx') |
|
121 | 155 | >>> import sphinx |
|
122 | 156 | >>> modpath = sphinx.__path__[0] |
|
123 | 157 | >>> res = docwriter._uri2path('sphinx.builder') |
|
124 | 158 | >>> res == os.path.join(modpath, 'builder.py') |
|
125 | 159 | True |
|
126 | 160 | >>> res = docwriter._uri2path('sphinx') |
|
127 | 161 | >>> res == os.path.join(modpath, '__init__.py') |
|
128 | 162 | True |
|
129 | 163 | >>> docwriter._uri2path('sphinx.does_not_exist') |
|
130 | 164 | |
|
131 | 165 | ''' |
|
132 | 166 | if uri == self.package_name: |
|
133 | 167 | return os.path.join(self.root_path, '__init__.py') |
|
134 | 168 | path = uri.replace('.', os.path.sep) |
|
135 | 169 | path = path.replace(self.package_name + os.path.sep, '') |
|
136 | 170 | path = os.path.join(self.root_path, path) |
|
137 | 171 | # XXX maybe check for extensions as well? |
|
138 | 172 | if os.path.exists(path + '.py'): # file |
|
139 | 173 | path += '.py' |
|
140 | 174 | elif os.path.exists(os.path.join(path, '__init__.py')): |
|
141 | 175 | path = os.path.join(path, '__init__.py') |
|
142 | 176 | else: |
|
143 | 177 | return None |
|
144 | 178 | return path |
|
145 | 179 | |
|
146 | 180 | def _path2uri(self, dirpath): |
|
147 | 181 | ''' Convert directory path to uri ''' |
|
148 | 182 | relpath = dirpath.replace(self.root_path, self.package_name) |
|
149 | 183 | if relpath.startswith(os.path.sep): |
|
150 | 184 | relpath = relpath[1:] |
|
151 | 185 | return relpath.replace(os.path.sep, '.') |
|
152 | 186 | |
|
153 | 187 | def _parse_module(self, uri): |
|
154 | 188 | ''' Parse module defined in *uri* ''' |
|
155 | 189 | filename = self._uri2path(uri) |
|
156 | 190 | if filename is None: |
|
157 | 191 | # nothing that we could handle here. |
|
158 | 192 | return ([],[]) |
|
159 | 193 | with open(filename, 'rb') as f: |
|
160 | 194 | mod = ast.parse(f.read()) |
|
161 |
return |
|
|
162 | ||
|
163 | @staticmethod | |
|
164 | def _find_functions_classes(mod): | |
|
165 | """Extract top-level functions and classes from a module AST. | |
|
166 | ||
|
167 | Skips objects with an @undoc decorator, or a name starting with '_'. | |
|
168 | """ | |
|
169 | def has_undoc_decorator(node): | |
|
170 | return any(isinstance(d, ast.Name) and d.id == 'undoc' \ | |
|
171 | for d in node.decorator_list) | |
|
172 | ||
|
173 | functions, classes = [], [] | |
|
174 | for node in mod.body: | |
|
175 | if isinstance(node, ast.FunctionDef) and \ | |
|
176 | not node.name.startswith('_') and \ | |
|
177 | not has_undoc_decorator(node): | |
|
178 | functions.append(node.name) | |
|
179 | elif isinstance(node, ast.ClassDef) and \ | |
|
180 | not node.name.startswith('_') and \ | |
|
181 | not has_undoc_decorator(node): | |
|
182 | cls = Obj(name=node.name) | |
|
183 | cls.has_init = any(isinstance(n, ast.FunctionDef) and \ | |
|
184 | n.name=='__init__' for n in node.body) | |
|
185 | classes.append(cls) | |
|
186 | ||
|
187 | return functions, classes | |
|
195 | return FuncClsScanner().scan(mod) | |
|
188 | 196 | |
|
189 | 197 | def generate_api_doc(self, uri): |
|
190 | 198 | '''Make autodoc documentation template string for a module |
|
191 | 199 | |
|
192 | 200 | Parameters |
|
193 | 201 | ---------- |
|
194 | 202 | uri : string |
|
195 | 203 | python location of module - e.g 'sphinx.builder' |
|
196 | 204 | |
|
197 | 205 | Returns |
|
198 | 206 | ------- |
|
199 | 207 | S : string |
|
200 | 208 | Contents of API doc |
|
201 | 209 | ''' |
|
202 | 210 | # get the names of all classes and functions |
|
203 | 211 | functions, classes = self._parse_module(uri) |
|
204 | 212 | if not len(functions) and not len(classes): |
|
205 | 213 | print 'WARNING: Empty -',uri # dbg |
|
206 | 214 | return '' |
|
207 | 215 | |
|
208 | 216 | # Make a shorter version of the uri that omits the package name for |
|
209 | 217 | # titles |
|
210 | 218 | uri_short = re.sub(r'^%s\.' % self.package_name,'',uri) |
|
211 | 219 | |
|
212 | 220 | ad = '.. AUTO-GENERATED FILE -- DO NOT EDIT!\n\n' |
|
213 | 221 | |
|
214 | 222 | # Set the chapter title to read 'Module:' for all modules except for the |
|
215 | 223 | # main packages |
|
216 | 224 | if '.' in uri: |
|
217 | 225 | chap_title = 'Module: :mod:`' + uri_short + '`' |
|
218 | 226 | else: |
|
219 | 227 | chap_title = ':mod:`' + uri_short + '`' |
|
220 | 228 | ad += chap_title + '\n' + self.rst_section_levels[1] * len(chap_title) |
|
221 | 229 | |
|
222 | 230 | ad += '\n.. automodule:: ' + uri + '\n' |
|
223 | 231 | ad += '\n.. currentmodule:: ' + uri + '\n' |
|
224 | 232 | |
|
225 | 233 | if classes: |
|
226 | 234 | subhead = str(len(classes)) + (' Classes' if len(classes) > 1 else ' Class') |
|
227 | 235 | ad += '\n'+ subhead + '\n' + \ |
|
228 | 236 | self.rst_section_levels[2] * len(subhead) + '\n' |
|
229 | 237 | |
|
230 | 238 | for c in classes: |
|
231 | 239 | ad += '\n.. autoclass:: ' + c.name + '\n' |
|
232 | 240 | # must NOT exclude from index to keep cross-refs working |
|
233 | 241 | ad += ' :members:\n' \ |
|
234 | 242 | ' :show-inheritance:\n' |
|
235 | 243 | if c.has_init: |
|
236 | 244 | ad += '\n .. automethod:: __init__\n' |
|
237 | 245 | |
|
238 | 246 | if functions: |
|
239 | 247 | subhead = str(len(functions)) + (' Functions' if len(functions) > 1 else ' Function') |
|
240 | 248 | ad += '\n'+ subhead + '\n' + \ |
|
241 | 249 | self.rst_section_levels[2] * len(subhead) + '\n' |
|
242 | 250 | for f in functions: |
|
243 | 251 | # must NOT exclude from index to keep cross-refs working |
|
244 | 252 | ad += '\n.. autofunction:: ' + uri + '.' + f + '\n\n' |
|
245 | 253 | return ad |
|
246 | 254 | |
|
247 | 255 | def _survives_exclude(self, matchstr, match_type): |
|
248 | 256 | ''' Returns True if *matchstr* does not match patterns |
|
249 | 257 | |
|
250 | 258 | ``self.package_name`` removed from front of string if present |
|
251 | 259 | |
|
252 | 260 | Examples |
|
253 | 261 | -------- |
|
254 | 262 | >>> dw = ApiDocWriter('sphinx') |
|
255 | 263 | >>> dw._survives_exclude('sphinx.okpkg', 'package') |
|
256 | 264 | True |
|
257 | 265 | >>> dw.package_skip_patterns.append('^\\.badpkg$') |
|
258 | 266 | >>> dw._survives_exclude('sphinx.badpkg', 'package') |
|
259 | 267 | False |
|
260 | 268 | >>> dw._survives_exclude('sphinx.badpkg', 'module') |
|
261 | 269 | True |
|
262 | 270 | >>> dw._survives_exclude('sphinx.badmod', 'module') |
|
263 | 271 | True |
|
264 | 272 | >>> dw.module_skip_patterns.append('^\\.badmod$') |
|
265 | 273 | >>> dw._survives_exclude('sphinx.badmod', 'module') |
|
266 | 274 | False |
|
267 | 275 | ''' |
|
268 | 276 | if match_type == 'module': |
|
269 | 277 | patterns = self.module_skip_patterns |
|
270 | 278 | elif match_type == 'package': |
|
271 | 279 | patterns = self.package_skip_patterns |
|
272 | 280 | else: |
|
273 | 281 | raise ValueError('Cannot interpret match type "%s"' |
|
274 | 282 | % match_type) |
|
275 | 283 | # Match to URI without package name |
|
276 | 284 | L = len(self.package_name) |
|
277 | 285 | if matchstr[:L] == self.package_name: |
|
278 | 286 | matchstr = matchstr[L:] |
|
279 | 287 | for pat in patterns: |
|
280 | 288 | try: |
|
281 | 289 | pat.search |
|
282 | 290 | except AttributeError: |
|
283 | 291 | pat = re.compile(pat) |
|
284 | 292 | if pat.search(matchstr): |
|
285 | 293 | return False |
|
286 | 294 | return True |
|
287 | 295 | |
|
288 | 296 | def discover_modules(self): |
|
289 | 297 | ''' Return module sequence discovered from ``self.package_name`` |
|
290 | 298 | |
|
291 | 299 | |
|
292 | 300 | Parameters |
|
293 | 301 | ---------- |
|
294 | 302 | None |
|
295 | 303 | |
|
296 | 304 | Returns |
|
297 | 305 | ------- |
|
298 | 306 | mods : sequence |
|
299 | 307 | Sequence of module names within ``self.package_name`` |
|
300 | 308 | |
|
301 | 309 | Examples |
|
302 | 310 | -------- |
|
303 | 311 | >>> dw = ApiDocWriter('sphinx') |
|
304 | 312 | >>> mods = dw.discover_modules() |
|
305 | 313 | >>> 'sphinx.util' in mods |
|
306 | 314 | True |
|
307 | 315 | >>> dw.package_skip_patterns.append('\.util$') |
|
308 | 316 | >>> 'sphinx.util' in dw.discover_modules() |
|
309 | 317 | False |
|
310 | 318 | >>> |
|
311 | 319 | ''' |
|
312 | 320 | modules = [self.package_name] |
|
313 | 321 | # raw directory parsing |
|
314 | 322 | for dirpath, dirnames, filenames in os.walk(self.root_path): |
|
315 | 323 | # Check directory names for packages |
|
316 | 324 | root_uri = self._path2uri(os.path.join(self.root_path, |
|
317 | 325 | dirpath)) |
|
318 | 326 | for dirname in dirnames[:]: # copy list - we modify inplace |
|
319 | 327 | package_uri = '.'.join((root_uri, dirname)) |
|
320 | 328 | if (self._uri2path(package_uri) and |
|
321 | 329 | self._survives_exclude(package_uri, 'package')): |
|
322 | 330 | modules.append(package_uri) |
|
323 | 331 | else: |
|
324 | 332 | dirnames.remove(dirname) |
|
325 | 333 | # Check filenames for modules |
|
326 | 334 | for filename in filenames: |
|
327 | 335 | module_name = filename[:-3] |
|
328 | 336 | module_uri = '.'.join((root_uri, module_name)) |
|
329 | 337 | if (self._uri2path(module_uri) and |
|
330 | 338 | self._survives_exclude(module_uri, 'module')): |
|
331 | 339 | modules.append(module_uri) |
|
332 | 340 | return sorted(modules) |
|
333 | 341 | |
|
334 | 342 | def write_modules_api(self, modules,outdir): |
|
335 | 343 | # write the list |
|
336 | 344 | written_modules = [] |
|
337 | 345 | for m in modules: |
|
338 | 346 | api_str = self.generate_api_doc(m) |
|
339 | 347 | if not api_str: |
|
340 | 348 | continue |
|
341 | 349 | # write out to file |
|
342 | 350 | outfile = os.path.join(outdir, |
|
343 | 351 | m + self.rst_extension) |
|
344 | 352 | fileobj = open(outfile, 'wt') |
|
345 | 353 | fileobj.write(api_str) |
|
346 | 354 | fileobj.close() |
|
347 | 355 | written_modules.append(m) |
|
348 | 356 | self.written_modules = written_modules |
|
349 | 357 | |
|
350 | 358 | def write_api_docs(self, outdir): |
|
351 | 359 | """Generate API reST files. |
|
352 | 360 | |
|
353 | 361 | Parameters |
|
354 | 362 | ---------- |
|
355 | 363 | outdir : string |
|
356 | 364 | Directory name in which to store files |
|
357 | 365 | We create automatic filenames for each module |
|
358 | 366 | |
|
359 | 367 | Returns |
|
360 | 368 | ------- |
|
361 | 369 | None |
|
362 | 370 | |
|
363 | 371 | Notes |
|
364 | 372 | ----- |
|
365 | 373 | Sets self.written_modules to list of written modules |
|
366 | 374 | """ |
|
367 | 375 | if not os.path.exists(outdir): |
|
368 | 376 | os.mkdir(outdir) |
|
369 | 377 | # compose list of modules |
|
370 | 378 | modules = self.discover_modules() |
|
371 | 379 | self.write_modules_api(modules,outdir) |
|
372 | 380 | |
|
373 | 381 | def write_index(self, outdir, froot='gen', relative_to=None): |
|
374 | 382 | """Make a reST API index file from written files |
|
375 | 383 | |
|
376 | 384 | Parameters |
|
377 | 385 | ---------- |
|
378 | 386 | path : string |
|
379 | 387 | Filename to write index to |
|
380 | 388 | outdir : string |
|
381 | 389 | Directory to which to write generated index file |
|
382 | 390 | froot : string, optional |
|
383 | 391 | root (filename without extension) of filename to write to |
|
384 | 392 | Defaults to 'gen'. We add ``self.rst_extension``. |
|
385 | 393 | relative_to : string |
|
386 | 394 | path to which written filenames are relative. This |
|
387 | 395 | component of the written file path will be removed from |
|
388 | 396 | outdir, in the generated index. Default is None, meaning, |
|
389 | 397 | leave path as it is. |
|
390 | 398 | """ |
|
391 | 399 | if self.written_modules is None: |
|
392 | 400 | raise ValueError('No modules written') |
|
393 | 401 | # Get full filename path |
|
394 | 402 | path = os.path.join(outdir, froot+self.rst_extension) |
|
395 | 403 | # Path written into index is relative to rootpath |
|
396 | 404 | if relative_to is not None: |
|
397 | 405 | relpath = outdir.replace(relative_to + os.path.sep, '') |
|
398 | 406 | else: |
|
399 | 407 | relpath = outdir |
|
400 | 408 | idx = open(path,'wt') |
|
401 | 409 | w = idx.write |
|
402 | 410 | w('.. AUTO-GENERATED FILE -- DO NOT EDIT!\n\n') |
|
403 | 411 | w('.. toctree::\n\n') |
|
404 | 412 | for f in self.written_modules: |
|
405 | 413 | w(' %s\n' % os.path.join(relpath,f)) |
|
406 | 414 | idx.close() |
General Comments 0
You need to be logged in to leave comments.
Login now