##// END OF EJS Templates
Python 3 print functions...
MinRK -
Show More
@@ -1,66 +1,66
1 1 #!/usr/bin/env python
2 2 """Script to auto-generate our API docs.
3 3 """
4 4 # stdlib imports
5 5 import os
6 6 import sys
7 7
8 8 # local imports
9 9 sys.path.append(os.path.abspath('sphinxext'))
10 10 from apigen import ApiDocWriter
11 11
12 12 #*****************************************************************************
13 13 if __name__ == '__main__':
14 14 pjoin = os.path.join
15 15 package = 'IPython'
16 16 outdir = pjoin('source','api','generated')
17 17 docwriter = ApiDocWriter(package,rst_extension='.txt')
18 18 # You have to escape the . here because . is a special char for regexps.
19 19 # You must do make clean if you change this!
20 20 docwriter.package_skip_patterns += [r'\.fixes$',
21 21 r'\.external$',
22 22 r'\.extensions',
23 23 r'\.kernel\.config',
24 24 r'\.attic',
25 25 r'\.quarantine',
26 26 r'\.deathrow',
27 27 r'\.config\.default',
28 28 r'\.config\.profile',
29 29 r'\.frontend',
30 30 r'\.gui',
31 31 r'\.kernel',
32 32 # For now, the zmq code has
33 33 # unconditional top-level code so it's
34 34 # not import safe. This needs fixing
35 35 r'\.zmq',
36 36 ]
37 37
38 38 docwriter.module_skip_patterns += [ r'\.core\.fakemodule',
39 39 r'\.testing\.iptest',
40 40 # Keeping these disabled is OK
41 41 r'\.parallel\.controller\.mongodb',
42 42 r'\.lib\.inputhookwx',
43 43 r'\.lib\.inputhookgtk',
44 44 r'\.cocoa',
45 45 r'\.ipdoctest',
46 46 r'\.Gnuplot',
47 47 r'\.frontend\.process\.winprocess',
48 48 r'\.frontend',
49 49 r'\.Shell',
50 50 ]
51 51
52 52 # If we don't have pexpect, we can't load irunner, so skip any code that
53 53 # depends on it
54 54 try:
55 55 import pexpect
56 56 except ImportError:
57 57 docwriter.module_skip_patterns += [r'\.lib\.irunner',
58 58 r'\.testing\.mkdoctests']
59 59 # Now, generate the outputs
60 60 docwriter.write_api_docs(outdir)
61 61 # Write index with .rst extension - we can include it, but Sphinx won't try
62 62 # to compile it
63 63 docwriter.write_index(outdir, 'gen.rst',
64 64 relative_to = pjoin('source','api')
65 65 )
66 print '%d files written' % len(docwriter.written_modules)
66 print ('%d files written' % len(docwriter.written_modules))
@@ -1,411 +1,413
1 1 """Attempt to generate templates for module reference with Sphinx
2 2
3 3 XXX - we exclude extension modules
4 4
5 5 To include extension modules, first identify them as valid in the
6 6 ``_uri2path`` method, then handle them in the ``_parse_module`` script.
7 7
8 8 We get functions and classes by parsing the text of .py files.
9 9 Alternatively we could import the modules for discovery, and we'd have
10 10 to do that for extension modules. This would involve changing the
11 11 ``_parse_module`` method to work via import and introspection, and
12 12 might involve changing ``discover_modules`` (which determines which
13 13 files are modules, and therefore which module URIs will be passed to
14 14 ``_parse_module``).
15 15
16 16 NOTE: this is a modified version of a script originally shipped with the
17 17 PyMVPA project, which we've adapted for NIPY use. PyMVPA is an MIT-licensed
18 18 project."""
19 19
20 from __future__ import print_function
21
20 22 # Stdlib imports
21 23 import ast
22 24 import os
23 25 import re
24 26
25 27 class Obj(object):
26 28 '''Namespace to hold arbitrary information.'''
27 29 def __init__(self, **kwargs):
28 30 for k, v in kwargs.items():
29 31 setattr(self, k, v)
30 32
31 33 class FuncClsScanner(ast.NodeVisitor):
32 34 """Scan a module for top-level functions and classes.
33 35
34 36 Skips objects with an @undoc decorator, or a name starting with '_'.
35 37 """
36 38 def __init__(self):
37 39 ast.NodeVisitor.__init__(self)
38 40 self.classes = []
39 41 self.classes_seen = set()
40 42 self.functions = []
41 43
42 44 @staticmethod
43 45 def has_undoc_decorator(node):
44 46 return any(isinstance(d, ast.Name) and d.id == 'undoc' \
45 47 for d in node.decorator_list)
46 48
47 49 def visit_FunctionDef(self, node):
48 50 if not (node.name.startswith('_') or self.has_undoc_decorator(node)) \
49 51 and node.name not in self.functions:
50 52 self.functions.append(node.name)
51 53
52 54 def visit_ClassDef(self, node):
53 55 if not (node.name.startswith('_') or self.has_undoc_decorator(node)) \
54 56 and node.name not in self.classes_seen:
55 57 cls = Obj(name=node.name)
56 58 cls.has_init = any(isinstance(n, ast.FunctionDef) and \
57 59 n.name=='__init__' for n in node.body)
58 60 self.classes.append(cls)
59 61 self.classes_seen.add(node.name)
60 62
61 63 def scan(self, mod):
62 64 self.visit(mod)
63 65 return self.functions, self.classes
64 66
65 67 # Functions and classes
66 68 class ApiDocWriter(object):
67 69 ''' Class for automatic detection and parsing of API docs
68 70 to Sphinx-parsable reST format'''
69 71
70 72 # only separating first two levels
71 73 rst_section_levels = ['*', '=', '-', '~', '^']
72 74
73 75 def __init__(self,
74 76 package_name,
75 77 rst_extension='.rst',
76 78 package_skip_patterns=None,
77 79 module_skip_patterns=None,
78 80 ):
79 81 ''' Initialize package for parsing
80 82
81 83 Parameters
82 84 ----------
83 85 package_name : string
84 86 Name of the top-level package. *package_name* must be the
85 87 name of an importable package
86 88 rst_extension : string, optional
87 89 Extension for reST files, default '.rst'
88 90 package_skip_patterns : None or sequence of {strings, regexps}
89 91 Sequence of strings giving URIs of packages to be excluded
90 92 Operates on the package path, starting at (including) the
91 93 first dot in the package path, after *package_name* - so,
92 94 if *package_name* is ``sphinx``, then ``sphinx.util`` will
93 95 result in ``.util`` being passed for earching by these
94 96 regexps. If is None, gives default. Default is:
95 97 ['\.tests$']
96 98 module_skip_patterns : None or sequence
97 99 Sequence of strings giving URIs of modules to be excluded
98 100 Operates on the module name including preceding URI path,
99 101 back to the first dot after *package_name*. For example
100 102 ``sphinx.util.console`` results in the string to search of
101 103 ``.util.console``
102 104 If is None, gives default. Default is:
103 105 ['\.setup$', '\._']
104 106 '''
105 107 if package_skip_patterns is None:
106 108 package_skip_patterns = ['\\.tests$']
107 109 if module_skip_patterns is None:
108 110 module_skip_patterns = ['\\.setup$', '\\._']
109 111 self.package_name = package_name
110 112 self.rst_extension = rst_extension
111 113 self.package_skip_patterns = package_skip_patterns
112 114 self.module_skip_patterns = module_skip_patterns
113 115
114 116 def get_package_name(self):
115 117 return self._package_name
116 118
117 119 def set_package_name(self, package_name):
118 120 ''' Set package_name
119 121
120 122 >>> docwriter = ApiDocWriter('sphinx')
121 123 >>> import sphinx
122 124 >>> docwriter.root_path == sphinx.__path__[0]
123 125 True
124 126 >>> docwriter.package_name = 'docutils'
125 127 >>> import docutils
126 128 >>> docwriter.root_path == docutils.__path__[0]
127 129 True
128 130 '''
129 131 # It's also possible to imagine caching the module parsing here
130 132 self._package_name = package_name
131 133 self.root_module = __import__(package_name)
132 134 self.root_path = self.root_module.__path__[0]
133 135 self.written_modules = None
134 136
135 137 package_name = property(get_package_name, set_package_name, None,
136 138 'get/set package_name')
137 139
138 140 def _uri2path(self, uri):
139 141 ''' Convert uri to absolute filepath
140 142
141 143 Parameters
142 144 ----------
143 145 uri : string
144 146 URI of python module to return path for
145 147
146 148 Returns
147 149 -------
148 150 path : None or string
149 151 Returns None if there is no valid path for this URI
150 152 Otherwise returns absolute file system path for URI
151 153
152 154 Examples
153 155 --------
154 156 >>> docwriter = ApiDocWriter('sphinx')
155 157 >>> import sphinx
156 158 >>> modpath = sphinx.__path__[0]
157 159 >>> res = docwriter._uri2path('sphinx.builder')
158 160 >>> res == os.path.join(modpath, 'builder.py')
159 161 True
160 162 >>> res = docwriter._uri2path('sphinx')
161 163 >>> res == os.path.join(modpath, '__init__.py')
162 164 True
163 165 >>> docwriter._uri2path('sphinx.does_not_exist')
164 166
165 167 '''
166 168 if uri == self.package_name:
167 169 return os.path.join(self.root_path, '__init__.py')
168 170 path = uri.replace('.', os.path.sep)
169 171 path = path.replace(self.package_name + os.path.sep, '')
170 172 path = os.path.join(self.root_path, path)
171 173 # XXX maybe check for extensions as well?
172 174 if os.path.exists(path + '.py'): # file
173 175 path += '.py'
174 176 elif os.path.exists(os.path.join(path, '__init__.py')):
175 177 path = os.path.join(path, '__init__.py')
176 178 else:
177 179 return None
178 180 return path
179 181
180 182 def _path2uri(self, dirpath):
181 183 ''' Convert directory path to uri '''
182 184 relpath = dirpath.replace(self.root_path, self.package_name)
183 185 if relpath.startswith(os.path.sep):
184 186 relpath = relpath[1:]
185 187 return relpath.replace(os.path.sep, '.')
186 188
187 189 def _parse_module(self, uri):
188 190 ''' Parse module defined in *uri* '''
189 191 filename = self._uri2path(uri)
190 192 if filename is None:
191 193 # nothing that we could handle here.
192 194 return ([],[])
193 195 with open(filename, 'rb') as f:
194 196 mod = ast.parse(f.read())
195 197 return FuncClsScanner().scan(mod)
196 198
197 199 def generate_api_doc(self, uri):
198 200 '''Make autodoc documentation template string for a module
199 201
200 202 Parameters
201 203 ----------
202 204 uri : string
203 205 python location of module - e.g 'sphinx.builder'
204 206
205 207 Returns
206 208 -------
207 209 S : string
208 210 Contents of API doc
209 211 '''
210 212 # get the names of all classes and functions
211 213 functions, classes = self._parse_module(uri)
212 214 if not len(functions) and not len(classes):
213 print 'WARNING: Empty -',uri # dbg
215 print ('WARNING: Empty -', uri) # dbg
214 216 return ''
215 217
216 218 # Make a shorter version of the uri that omits the package name for
217 219 # titles
218 220 uri_short = re.sub(r'^%s\.' % self.package_name,'',uri)
219 221
220 222 ad = '.. AUTO-GENERATED FILE -- DO NOT EDIT!\n\n'
221 223
222 224 # Set the chapter title to read 'Module:' for all modules except for the
223 225 # main packages
224 226 if '.' in uri:
225 227 chap_title = 'Module: :mod:`' + uri_short + '`'
226 228 else:
227 229 chap_title = ':mod:`' + uri_short + '`'
228 230 ad += chap_title + '\n' + self.rst_section_levels[1] * len(chap_title)
229 231
230 232 ad += '\n.. automodule:: ' + uri + '\n'
231 233 ad += '\n.. currentmodule:: ' + uri + '\n'
232 234
233 235 if classes:
234 236 subhead = str(len(classes)) + (' Classes' if len(classes) > 1 else ' Class')
235 237 ad += '\n'+ subhead + '\n' + \
236 238 self.rst_section_levels[2] * len(subhead) + '\n'
237 239
238 240 for c in classes:
239 241 ad += '\n.. autoclass:: ' + c.name + '\n'
240 242 # must NOT exclude from index to keep cross-refs working
241 243 ad += ' :members:\n' \
242 244 ' :show-inheritance:\n'
243 245 if c.has_init:
244 246 ad += '\n .. automethod:: __init__\n'
245 247
246 248 if functions:
247 249 subhead = str(len(functions)) + (' Functions' if len(functions) > 1 else ' Function')
248 250 ad += '\n'+ subhead + '\n' + \
249 251 self.rst_section_levels[2] * len(subhead) + '\n'
250 252 for f in functions:
251 253 # must NOT exclude from index to keep cross-refs working
252 254 ad += '\n.. autofunction:: ' + uri + '.' + f + '\n\n'
253 255 return ad
254 256
255 257 def _survives_exclude(self, matchstr, match_type):
256 258 ''' Returns True if *matchstr* does not match patterns
257 259
258 260 ``self.package_name`` removed from front of string if present
259 261
260 262 Examples
261 263 --------
262 264 >>> dw = ApiDocWriter('sphinx')
263 265 >>> dw._survives_exclude('sphinx.okpkg', 'package')
264 266 True
265 267 >>> dw.package_skip_patterns.append('^\\.badpkg$')
266 268 >>> dw._survives_exclude('sphinx.badpkg', 'package')
267 269 False
268 270 >>> dw._survives_exclude('sphinx.badpkg', 'module')
269 271 True
270 272 >>> dw._survives_exclude('sphinx.badmod', 'module')
271 273 True
272 274 >>> dw.module_skip_patterns.append('^\\.badmod$')
273 275 >>> dw._survives_exclude('sphinx.badmod', 'module')
274 276 False
275 277 '''
276 278 if match_type == 'module':
277 279 patterns = self.module_skip_patterns
278 280 elif match_type == 'package':
279 281 patterns = self.package_skip_patterns
280 282 else:
281 283 raise ValueError('Cannot interpret match type "%s"'
282 284 % match_type)
283 285 # Match to URI without package name
284 286 L = len(self.package_name)
285 287 if matchstr[:L] == self.package_name:
286 288 matchstr = matchstr[L:]
287 289 for pat in patterns:
288 290 try:
289 291 pat.search
290 292 except AttributeError:
291 293 pat = re.compile(pat)
292 294 if pat.search(matchstr):
293 295 return False
294 296 return True
295 297
296 298 def discover_modules(self):
297 299 ''' Return module sequence discovered from ``self.package_name``
298 300
299 301
300 302 Parameters
301 303 ----------
302 304 None
303 305
304 306 Returns
305 307 -------
306 308 mods : sequence
307 309 Sequence of module names within ``self.package_name``
308 310
309 311 Examples
310 312 --------
311 313 >>> dw = ApiDocWriter('sphinx')
312 314 >>> mods = dw.discover_modules()
313 315 >>> 'sphinx.util' in mods
314 316 True
315 317 >>> dw.package_skip_patterns.append('\.util$')
316 318 >>> 'sphinx.util' in dw.discover_modules()
317 319 False
318 320 >>>
319 321 '''
320 322 modules = [self.package_name]
321 323 # raw directory parsing
322 324 for dirpath, dirnames, filenames in os.walk(self.root_path):
323 325 # Check directory names for packages
324 326 root_uri = self._path2uri(os.path.join(self.root_path,
325 327 dirpath))
326 328 for dirname in dirnames[:]: # copy list - we modify inplace
327 329 package_uri = '.'.join((root_uri, dirname))
328 330 if (self._uri2path(package_uri) and
329 331 self._survives_exclude(package_uri, 'package')):
330 332 modules.append(package_uri)
331 333 else:
332 334 dirnames.remove(dirname)
333 335 # Check filenames for modules
334 336 for filename in filenames:
335 337 module_name = filename[:-3]
336 338 module_uri = '.'.join((root_uri, module_name))
337 339 if (self._uri2path(module_uri) and
338 340 self._survives_exclude(module_uri, 'module')):
339 341 modules.append(module_uri)
340 342 return sorted(modules)
341 343
342 344 def write_modules_api(self, modules,outdir):
343 345 # write the list
344 346 written_modules = []
345 347 for m in modules:
346 348 api_str = self.generate_api_doc(m)
347 349 if not api_str:
348 350 continue
349 351 # write out to file
350 352 outfile = os.path.join(outdir,
351 353 m + self.rst_extension)
352 354 fileobj = open(outfile, 'wt')
353 355 fileobj.write(api_str)
354 356 fileobj.close()
355 357 written_modules.append(m)
356 358 self.written_modules = written_modules
357 359
358 360 def write_api_docs(self, outdir):
359 361 """Generate API reST files.
360 362
361 363 Parameters
362 364 ----------
363 365 outdir : string
364 366 Directory name in which to store files
365 367 We create automatic filenames for each module
366 368
367 369 Returns
368 370 -------
369 371 None
370 372
371 373 Notes
372 374 -----
373 375 Sets self.written_modules to list of written modules
374 376 """
375 377 if not os.path.exists(outdir):
376 378 os.mkdir(outdir)
377 379 # compose list of modules
378 380 modules = self.discover_modules()
379 381 self.write_modules_api(modules,outdir)
380 382
381 383 def write_index(self, outdir, path='gen.rst', relative_to=None):
382 384 """Make a reST API index file from written files
383 385
384 386 Parameters
385 387 ----------
386 388 outdir : string
387 389 Directory to which to write generated index file
388 390 path : string
389 391 Filename to write index to
390 392 relative_to : string
391 393 path to which written filenames are relative. This
392 394 component of the written file path will be removed from
393 395 outdir, in the generated index. Default is None, meaning,
394 396 leave path as it is.
395 397 """
396 398 if self.written_modules is None:
397 399 raise ValueError('No modules written')
398 400 # Get full filename path
399 401 path = os.path.join(outdir, path)
400 402 # Path written into index is relative to rootpath
401 403 if relative_to is not None:
402 404 relpath = outdir.replace(relative_to + os.path.sep, '')
403 405 else:
404 406 relpath = outdir
405 407 idx = open(path,'wt')
406 408 w = idx.write
407 409 w('.. AUTO-GENERATED FILE -- DO NOT EDIT!\n\n')
408 410 w('.. toctree::\n\n')
409 411 for f in self.written_modules:
410 412 w(' %s\n' % os.path.join(relpath,f))
411 413 idx.close()
General Comments 0
You need to be logged in to leave comments. Login now