source: trunk/adm/website/sphinxext/apigen.py @ 36

Last change on this file since 36 was 36, checked in by pinsard, 13 years ago

add end users website production
File size: 15.3 KB
Line 
1"""Attempt to generate templates for module reference with Sphinx
2
3XXX - we exclude extension modules
4
5To include extension modules, first identify them as valid in the
6``_uri2path`` method, then handle them in the ``_parse_module`` script.
7
8We get functions and classes by parsing the text of .py files.
9Alternatively we could import the modules for discovery, and we'd have
10to do that for extension modules.  This would involve changing the
11``_parse_module`` method to work via import and introspection, and
12might involve changing ``discover_modules`` (which determines which
13files are modules, and therefore which module URIs will be passed to
14``_parse_module``).
15
16NOTE: this is a modified version of a script originally shipped with the
17PyMVPA project, which we've adapted for NIPY use.  PyMVPA is an MIT-licensed
18project."""
19
20# Stdlib imports
21import os
22import re
23
24# Functions and classes
25class ApiDocWriter(object):
26    ''' Class for automatic detection and parsing of API docs
27    to Sphinx-parsable reST format'''
28
29    # only separating first two levels
30    rst_section_levels = ['*', '=', '-', '~', '^']
31
32    def __init__(self,
33                 package_name,
34                 rst_extension='.rst',
35                 package_skip_patterns=None,
36                 module_skip_patterns=None,
37                 ):
38        ''' Initialize package for parsing
39
40        Parameters
41        ----------
42        package_name : string
43            Name of the top-level package.  *package_name* must be the
44            name of an importable package
45        rst_extension : string, optional
46            Extension for reST files, default '.rst'
47        package_skip_patterns : None or sequence of {strings, regexps}
48            Sequence of strings giving URIs of packages to be excluded
49            Operates on the package path, starting at (including) the
50            first dot in the package path, after *package_name* - so,
51            if *package_name* is ``sphinx``, then ``sphinx.util`` will
52            result in ``.util`` being passed for earching by these
53            regexps.  If is None, gives default. Default is:
54            ['\.tests$']
55        module_skip_patterns : None or sequence
56            Sequence of strings giving URIs of modules to be excluded
57            Operates on the module name including preceding URI path,
58            back to the first dot after *package_name*.  For example
59            ``sphinx.util.console`` results in the string to search of
60            ``.util.console``
61            If is None, gives default. Default is:
62            ['\.setup$', '\._']
63        '''
64        if package_skip_patterns is None:
65            package_skip_patterns = ['\\.tests$']
66        if module_skip_patterns is None:
67            module_skip_patterns = ['\\.setup$', '\\._']
68        self.package_name = package_name
69        self.rst_extension = rst_extension
70        self.package_skip_patterns = package_skip_patterns
71        self.module_skip_patterns = module_skip_patterns
72
73    def get_package_name(self):
74        return self._package_name
75
76    def set_package_name(self, package_name):
77        ''' Set package_name
78
79        >>> docwriter = ApiDocWriter('sphinx')
80        >>> import sphinx
81        >>> docwriter.root_path == sphinx.__path__[0]
82        True
83        >>> docwriter.package_name = 'docutils'
84        >>> import docutils
85        >>> docwriter.root_path == docutils.__path__[0]
86        True
87        '''
88        # It's also possible to imagine caching the module parsing here
89        self._package_name = package_name
90        self.root_module = __import__(package_name)
91        self.root_path = self.root_module.__path__[0]
92        self.written_modules = None
93
94    package_name = property(get_package_name, set_package_name, None,
95                            'get/set package_name')
96
97    def _get_object_name(self, line):
98        ''' Get second token in line
99        >>> docwriter = ApiDocWriter('sphinx')
100        >>> docwriter._get_object_name("  def func():  ")
101        'func'
102        >>> docwriter._get_object_name("  class Klass(object):  ")
103        'Klass'
104        >>> docwriter._get_object_name("  class Klass:  ")
105        'Klass'
106        '''
107        name = line.split()[1].split('(')[0].strip()
108        # in case we have classes which are not derived from object
109        # ie. old style classes
110        return name.rstrip(':')
111
112    def _uri2path(self, uri):
113        ''' Convert uri to absolute filepath
114
115        Parameters
116        ----------
117        uri : string
118            URI of python module to return path for
119
120        Returns
121        -------
122        path : None or string
123            Returns None if there is no valid path for this URI
124            Otherwise returns absolute file system path for URI
125
126        Examples
127        --------
128        >>> docwriter = ApiDocWriter('sphinx')
129        >>> import sphinx
130        >>> modpath = sphinx.__path__[0]
131        >>> res = docwriter._uri2path('sphinx.builder')
132        >>> res == os.path.join(modpath, 'builder.py')
133        True
134        >>> res = docwriter._uri2path('sphinx')
135        >>> res == os.path.join(modpath, '__init__.py')
136        True
137        >>> docwriter._uri2path('sphinx.does_not_exist')
138
139        '''
140        if uri == self.package_name:
141            return os.path.join(self.root_path, '__init__.py')
142        path = uri.replace('.', os.path.sep)
143        path = path.replace(self.package_name + os.path.sep, '')
144        path = os.path.join(self.root_path, path)
145        # XXX maybe check for extensions as well?
146        if os.path.exists(path + '.py'): # file
147            path += '.py'
148        elif os.path.exists(os.path.join(path, '__init__.py')):
149            path = os.path.join(path, '__init__.py')
150        else:
151            return None
152        return path
153
154    def _path2uri(self, dirpath):
155        ''' Convert directory path to uri '''
156        relpath = dirpath.replace(self.root_path, self.package_name)
157        if relpath.startswith(os.path.sep):
158            relpath = relpath[1:]
159        return relpath.replace(os.path.sep, '.')
160
161    def _parse_module(self, uri):
162        ''' Parse module defined in *uri* '''
163        filename = self._uri2path(uri)
164        if filename is None:
165            # nothing that we could handle here.
166            return ([],[])
167        f = open(filename, 'rt')
168        functions, classes = self._parse_lines(f)
169        f.close()
170        return functions, classes
171   
172    def _parse_lines(self, linesource):
173        ''' Parse lines of text for functions and classes '''
174        functions = []
175        classes = []
176        for line in linesource:
177            if line.startswith('def ') and line.count('('):
178                # exclude private stuff
179                name = self._get_object_name(line)
180                if not name.startswith('_'):
181                    functions.append(name)
182            elif line.startswith('class '):
183                # exclude private stuff
184                name = self._get_object_name(line)
185                if not name.startswith('_'):
186                    classes.append(name)
187            else:
188                pass
189        functions.sort()
190        classes.sort()
191        return functions, classes
192
193    def generate_api_doc(self, uri):
194        '''Make autodoc documentation template string for a module
195
196        Parameters
197        ----------
198        uri : string
199            python location of module - e.g 'sphinx.builder'
200
201        Returns
202        -------
203        S : string
204            Contents of API doc
205        '''
206        # get the names of all classes and functions
207        functions, classes = self._parse_module(uri)
208        if not len(functions) and not len(classes):
209            print 'WARNING: Empty -',uri  # dbg
210            return ''
211
212        # Make a shorter version of the uri that omits the package name for
213        # titles
214        uri_short = re.sub(r'^%s\.' % self.package_name,'',uri)
215       
216        ad = '.. AUTO-GENERATED FILE -- DO NOT EDIT!\n\n'
217
218        chap_title = uri_short
219        ad += (chap_title+'\n'+ self.rst_section_levels[1] * len(chap_title)
220               + '\n\n')
221
222        # Set the chapter title to read 'module' for all modules except for the
223        # main packages
224        if '.' in uri:
225            title = 'Module: :mod:`' + uri_short + '`'
226        else:
227            title = ':mod:`' + uri_short + '`'
228        ad += title + '\n' + self.rst_section_levels[2] * len(title)
229
230        if len(classes):
231            ad += '\nInheritance diagram for ``%s``:\n\n' % uri
232            ad += '.. inheritance-diagram:: %s \n' % uri
233            ad += '   :parts: 3\n'
234
235        ad += '\n.. automodule:: ' + uri + '\n'
236        ad += '\n.. currentmodule:: ' + uri + '\n'
237        multi_class = len(classes) > 1
238        multi_fx = len(functions) > 1
239        if multi_class:
240            ad += '\n' + 'Classes' + '\n' + \
241                  self.rst_section_levels[2] * 7 + '\n'
242        elif len(classes) and multi_fx:
243            ad += '\n' + 'Class' + '\n' + \
244                  self.rst_section_levels[2] * 5 + '\n'
245        for c in classes:
246            ad += '\n:class:`' + c + '`\n' \
247                  + self.rst_section_levels[multi_class + 2 ] * \
248                  (len(c)+9) + '\n\n'
249            ad += '\n.. autoclass:: ' + c + '\n'
250            # must NOT exclude from index to keep cross-refs working
251            ad += '  :members:\n' \
252                  '  :undoc-members:\n' \
253                  '  :show-inheritance:\n' \
254                  '  :inherited-members:\n' \
255                  '\n' \
256                  '  .. automethod:: __init__\n'
257        if multi_fx:
258            ad += '\n' + 'Functions' + '\n' + \
259                  self.rst_section_levels[2] * 9 + '\n\n'
260        elif len(functions) and multi_class:
261            ad += '\n' + 'Function' + '\n' + \
262                  self.rst_section_levels[2] * 8 + '\n\n'
263        for f in functions:
264            # must NOT exclude from index to keep cross-refs working
265            ad += '\n.. autofunction:: ' + uri + '.' + f + '\n\n'
266        return ad
267
268    def _survives_exclude(self, matchstr, match_type):
269        ''' Returns True if *matchstr* does not match patterns
270
271        ``self.package_name`` removed from front of string if present
272
273        Examples
274        --------
275        >>> dw = ApiDocWriter('sphinx')
276        >>> dw._survives_exclude('sphinx.okpkg', 'package')
277        True
278        >>> dw.package_skip_patterns.append('^\\.badpkg$')
279        >>> dw._survives_exclude('sphinx.badpkg', 'package')
280        False
281        >>> dw._survives_exclude('sphinx.badpkg', 'module')
282        True
283        >>> dw._survives_exclude('sphinx.badmod', 'module')
284        True
285        >>> dw.module_skip_patterns.append('^\\.badmod$')
286        >>> dw._survives_exclude('sphinx.badmod', 'module')
287        False
288        '''
289        if match_type == 'module':
290            patterns = self.module_skip_patterns
291        elif match_type == 'package':
292            patterns = self.package_skip_patterns
293        else:
294            raise ValueError('Cannot interpret match type "%s"' 
295                             % match_type)
296        # Match to URI without package name
297        L = len(self.package_name)
298        if matchstr[:L] == self.package_name:
299            matchstr = matchstr[L:]
300        for pat in patterns:
301            try:
302                pat.search
303            except AttributeError:
304                pat = re.compile(pat)
305            if pat.search(matchstr):
306                return False
307        return True
308
309    def discover_modules(self):
310        ''' Return module sequence discovered from ``self.package_name``
311
312
313        Parameters
314        ----------
315        None
316
317        Returns
318        -------
319        mods : sequence
320            Sequence of module names within ``self.package_name``
321
322        Examples
323        --------
324        >>> dw = ApiDocWriter('sphinx')
325        >>> mods = dw.discover_modules()
326        >>> 'sphinx.util' in mods
327        True
328        >>> dw.package_skip_patterns.append('\.util$')
329        >>> 'sphinx.util' in dw.discover_modules()
330        False
331        >>>
332        '''
333        modules = [self.package_name]
334        # raw directory parsing
335        for dirpath, dirnames, filenames in os.walk(self.root_path):
336            # Check directory names for packages
337            root_uri = self._path2uri(os.path.join(self.root_path,
338                                                   dirpath))
339            for dirname in dirnames[:]: # copy list - we modify inplace
340                package_uri = '.'.join((root_uri, dirname))
341                if (self._uri2path(package_uri) and
342                    self._survives_exclude(package_uri, 'package')):
343                    modules.append(package_uri)
344                else:
345                    dirnames.remove(dirname)
346            # Check filenames for modules
347            for filename in filenames:
348                module_name = filename[:-3]
349                module_uri = '.'.join((root_uri, module_name))
350                if (self._uri2path(module_uri) and
351                    self._survives_exclude(module_uri, 'module')):
352                    modules.append(module_uri)
353        return sorted(modules)
354   
355    def write_modules_api(self, modules,outdir):
356        # write the list
357        written_modules = []
358        for m in modules:
359            api_str = self.generate_api_doc(m)
360            if not api_str:
361                continue
362            # write out to file
363            outfile = os.path.join(outdir,
364                                   m + self.rst_extension)
365            fileobj = open(outfile, 'wt')
366            fileobj.write(api_str)
367            fileobj.close()
368            written_modules.append(m)
369        self.written_modules = written_modules
370
371    def write_api_docs(self, outdir):
372        """Generate API reST files.
373
374        Parameters
375        ----------
376        outdir : string
377            Directory name in which to store files
378            We create automatic filenames for each module
379           
380        Returns
381        -------
382        None
383
384        Notes
385        -----
386        Sets self.written_modules to list of written modules
387        """
388        if not os.path.exists(outdir):
389            os.mkdir(outdir)
390        # compose list of modules
391        modules = self.discover_modules()
392        self.write_modules_api(modules,outdir)
393       
394    def write_index(self, outdir, froot='gen', relative_to=None):
395        """Make a reST API index file from written files
396
397        Parameters
398        ----------
399        path : string
400            Filename to write index to
401        outdir : string
402            Directory to which to write generated index file
403        froot : string, optional
404            root (filename without extension) of filename to write to
405            Defaults to 'gen'.  We add ``self.rst_extension``.
406        relative_to : string
407            path to which written filenames are relative.  This
408            component of the written file path will be removed from
409            outdir, in the generated index.  Default is None, meaning,
410            leave path as it is.
411        """
412        if self.written_modules is None:
413            raise ValueError('No modules written')
414        # Get full filename path
415        path = os.path.join(outdir, froot+self.rst_extension)
416        # Path written into index is relative to rootpath
417        if relative_to is not None:
418            relpath = outdir.replace(relative_to + os.path.sep, '')
419        else:
420            relpath = outdir
421        idx = open(path,'wt')
422        w = idx.write
423        w('.. AUTO-GENERATED FILE -- DO NOT EDIT!\n\n')
424        w('.. toctree::\n\n')
425        for f in self.written_modules:
426            w('   %s\n' % os.path.join(relpath,f))
427        idx.close()
Note: See TracBrowser for help on using the repository browser.