Source Code for Module ganeti.build.sphinx_ext

  1  # 
  2  # 
  3   
  4  # Copyright (C) 2011, 2012, 2013 Google Inc. 
  5  # All rights reserved. 
  6  # 
  7  # Redistribution and use in source and binary forms, with or without 
  8  # modification, are permitted provided that the following conditions are 
  9  # met: 
 10  # 
 11  # 1. Redistributions of source code must retain the above copyright notice, 
 12  # this list of conditions and the following disclaimer. 
 13  # 
 14  # 2. Redistributions in binary form must reproduce the above copyright 
 15  # notice, this list of conditions and the following disclaimer in the 
 16  # documentation and/or other materials provided with the distribution. 
 17  # 
 18  # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS 
 19  # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 
 20  # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 
 21  # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR 
 22  # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 
 23  # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 
 24  # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 
 25  # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 
 26  # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 
 27  # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 
 28  # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
 29   
 30   
 31  """Sphinx extension for building opcode documentation. 
 32   
 33  """ 
 34   
 35  import re 
 36  from cStringIO import StringIO 
 37   
 38  import docutils.statemachine 
 39  import docutils.nodes 
 40  import docutils.utils 
 41  import docutils.parsers.rst 
 42   
 43  import sphinx.errors 
 44  import sphinx.util.compat 
 45  import sphinx.roles 
 46  import sphinx.addnodes 
 47   
 48  s_compat = sphinx.util.compat 
 49   
 50  try: 
 51    # Access to a protected member of a client class 
 52    # pylint: disable=W0212 
 53    orig_manpage_role = docutils.parsers.rst.roles._roles["manpage"] 
 54  except (AttributeError, ValueError, KeyError), err: 
 55    # Normally the "manpage" role is registered by sphinx/roles.py 
 56    raise Exception("Can't find reST role named 'manpage': %s" % err) 
 57   
 58  from ganeti import _constants 
 59  from ganeti import constants 
 60  from ganeti import compat 
 61  from ganeti import errors 
 62  from ganeti import utils 
 63  from ganeti import opcodes 
 64  from ganeti import opcodes_base 
 65  from ganeti import ht 
 66  from ganeti import rapi 
 67  from ganeti import luxi 
 68  from ganeti import objects 
 69  from ganeti import http 
 70  from ganeti import pathutils 
 71   
 72  import ganeti.rapi.rlib2 # pylint: disable=W0611 
 73  import ganeti.rapi.connector # pylint: disable=W0611 
 74   
 75   
 76  #: Regular expression for man page names 
 77  _MAN_RE = re.compile(r"^(?P<name>[-\w_]+)\((?P<section>\d+)\)$") 
 78   
 79  _TAB_WIDTH = 2 
 80   
 81  RAPI_URI_ENCODE_RE = re.compile("[^_a-z0-9]+", re.I) 
 82   
 83   
 84 -class ReSTError(Exception): 
 85    """Custom class for generating errors in Sphinx. 
 86   
 87    """ 
 88   
 89   
 90 -def _GetCommonParamNames(): 
 91    """Builds a list of parameters common to all opcodes. 
 92   
 93    """ 
 94    names = set(map(compat.fst, opcodes.OpCode.OP_PARAMS)) 
 95   
 96    # The "depends" attribute should be listed 
 97    names.remove(opcodes_base.DEPEND_ATTR) 
 98   
 99    return names 
100   
101   
102  COMMON_PARAM_NAMES = _GetCommonParamNames() 
103   
104  #: Namespace for evaluating expressions 
105  EVAL_NS = dict(compat=compat, constants=constants, utils=utils, errors=errors, 
106                 rlib2=rapi.rlib2, luxi=luxi, rapi=rapi, objects=objects, 
107                 http=http, pathutils=pathutils) 
108   
109  # Constants documentation for man pages 
110  CV_ECODES_DOC = "ecodes" 
111  # We don't care about the leak of variables _, name and doc here. 
112  # pylint: disable=W0621 
113  CV_ECODES_DOC_LIST = [(name, doc) for (_, name, doc) in constants.CV_ALL_ECODES] 
114  DOCUMENTED_CONSTANTS = { 
115    CV_ECODES_DOC: CV_ECODES_DOC_LIST, 
116    } 
117   
118   
119 -class OpcodeError(sphinx.errors.SphinxError): 
120    category = "Opcode error" 
121   
122   
123 -def _SplitOption(text): 
124    """Split simple option list. 
125   
126    @type text: string 
127    @param text: Options, e.g. "foo, bar, baz" 
128   
129    """ 
130    return [i.strip(",").strip() for i in text.split()] 
131   
132   
133 -def _ParseAlias(text): 
134    """Parse simple assignment option. 
135   
136    @type text: string 
137    @param text: Assignments, e.g. "foo=bar, hello=world" 
138    @rtype: dict 
139   
140    """ 
141    result = {} 
142   
143    for part in _SplitOption(text): 
144      if "=" not in part: 
145        raise OpcodeError("Invalid option format, missing equal sign") 
146   
147      (name, value) = part.split("=", 1) 
148   
149      result[name.strip()] = value.strip() 
150   
151    return result 
152   
153   
154 -def _BuildOpcodeParams(op_id, include, exclude, alias): 
155    """Build opcode parameter documentation. 
156   
157    @type op_id: string 
158    @param op_id: Opcode ID 
159   
160    """ 
161    op_cls = opcodes.OP_MAPPING[op_id] 
162   
163    params_with_alias = \ 
164      utils.NiceSort([(alias.get(name, name), name, default, test, doc) 
165                      for (name, default, test, doc) in op_cls.GetAllParams()], 
166                     key=compat.fst) 
167   
168    for (rapi_name, name, default, test, doc) in params_with_alias: 
169      # Hide common parameters if not explicitly included 
170      if (name in COMMON_PARAM_NAMES and 
171          (not include or name not in include)): 
172        continue 
173      if exclude is not None and name in exclude: 
174        continue 
175      if include is not None and name not in include: 
176        continue 
177   
178      has_default = default is not None or default is not ht.NoDefault 
179      has_test = test is not None 
180   
181      buf = StringIO() 
182      buf.write("``%s``" % (rapi_name,)) 
183      if has_default or has_test: 
184        buf.write(" (") 
185        if has_default: 
186          if default == "": 
187            buf.write("defaults to the empty string") 
188          else: 
189            buf.write("defaults to ``%s``" % (default,)) 
190          if has_test: 
191            buf.write(", ") 
192        if has_test: 
193          buf.write("must be ``%s``" % (test,)) 
194        buf.write(")") 
195      yield buf.getvalue() 
196   
197      # Add text 
198      for line in doc.splitlines(): 
199        yield "  %s" % line 
200   
201   
202 -def _BuildOpcodeResult(op_id): 
203    """Build opcode result documentation. 
204   
205    @type op_id: string 
206    @param op_id: Opcode ID 
207   
208    """ 
209    op_cls = opcodes.OP_MAPPING[op_id] 
210   
211    result_fn = getattr(op_cls, "OP_RESULT", None) 
212   
213    if not result_fn: 
214      raise OpcodeError("Opcode '%s' has no result description" % op_id) 
215   
216    return "``%s``" % result_fn 
217   
218   
219 -class OpcodeParams(s_compat.Directive): 
220    """Custom directive for opcode parameters. 
221   
222    See also <http://docutils.sourceforge.net/docs/howto/rst-directives.html>. 
223   
224    """ 
225    has_content = False 
226    required_arguments = 1 
227    optional_arguments = 0 
228    final_argument_whitespace = False 
229    option_spec = dict(include=_SplitOption, exclude=_SplitOption, 
230                       alias=_ParseAlias) 
231   
232 -  def run(self): 
233      op_id = self.arguments[0] 
234      include = self.options.get("include", None) 
235      exclude = self.options.get("exclude", None) 
236      alias = self.options.get("alias", {}) 
237   
238      path = op_id 
239      include_text = "\n\n".join(_BuildOpcodeParams(op_id, 
240                                                    include, 
241                                                    exclude, 
242                                                    alias)) 
243   
244      # Inject into state machine 
245      include_lines = docutils.statemachine.string2lines(include_text, _TAB_WIDTH, 
246                                                         convert_whitespace=1) 
247      self.state_machine.insert_input(include_lines, path) 
248   
249      return [] 
250   
251   
252 -class OpcodeResult(s_compat.Directive): 
253    """Custom directive for opcode result. 
254   
255    See also <http://docutils.sourceforge.net/docs/howto/rst-directives.html>. 
256   
257    """ 
258    has_content = False 
259    required_arguments = 1 
260    optional_arguments = 0 
261    final_argument_whitespace = False 
262   
263 -  def run(self): 
264      op_id = self.arguments[0] 
265   
266      path = op_id 
267      include_text = _BuildOpcodeResult(op_id) 
268   
269      # Inject into state machine 
270      include_lines = docutils.statemachine.string2lines(include_text, _TAB_WIDTH, 
271                                                         convert_whitespace=1) 
272      self.state_machine.insert_input(include_lines, path) 
273   
274      return [] 
275   
276   
277 -def PythonEvalRole(role, rawtext, text, lineno, inliner, 
278                     options={}, content=[]): 
279    """Custom role to evaluate Python expressions. 
280   
281    The expression's result is included as a literal. 
282   
283    """ 
284    # pylint: disable=W0102,W0613,W0142 
285    # W0102: Dangerous default value as argument 
286    # W0142: Used * or ** magic 
287    # W0613: Unused argument 
288   
289    code = docutils.utils.unescape(text, restore_backslashes=True) 
290   
291    try: 
292      result = eval(code, EVAL_NS) 
293    except Exception, err: # pylint: disable=W0703 
294      msg = inliner.reporter.error("Failed to evaluate %r: %s" % (code, err), 
295                                   line=lineno) 
296      return ([inliner.problematic(rawtext, rawtext, msg)], [msg]) 
297   
298    node = docutils.nodes.literal("", unicode(result), **options) 
299   
300    return ([node], []) 
301   
302   
303 -class PythonAssert(s_compat.Directive): 
304    """Custom directive for writing assertions. 
305   
306    The content must be a valid Python expression. If its result does not 
307    evaluate to C{True}, the assertion fails. 
308   
309    """ 
310    has_content = True 
311    required_arguments = 0 
312    optional_arguments = 0 
313    final_argument_whitespace = False 
314   
315 -  def run(self): 
316      # Handle combinations of Sphinx and docutils not providing the wanted method 
317      if hasattr(self, "assert_has_content"): 
318        self.assert_has_content() 
319      else: 
320        assert self.content 
321   
322      code = "\n".join(self.content) 
323   
324      try: 
325        result = eval(code, EVAL_NS) 
326      except Exception, err: 
327        raise self.error("Failed to evaluate %r: %s" % (code, err)) 
328   
329      if not result: 
330        raise self.error("Assertion failed: %s" % (code, )) 
331   
332      return [] 
333   
334   
335 -def BuildQueryFields(fields): 
336    """Build query fields documentation. 
337   
338    @type fields: dict (field name as key, field details as value) 
339   
340    """ 
341    defs = [(fdef.name, fdef.doc) 
342             for (_, (fdef, _, _, _)) in utils.NiceSort(fields.items(), 
343                                                        key=compat.fst)] 
344    return BuildValuesDoc(defs) 
345   
346   
347 -def BuildValuesDoc(values): 
348    """Builds documentation for a list of values 
349   
350    @type values: list of tuples in the form (value, documentation) 
351   
352    """ 
353    for name, doc in values: 
354      assert len(doc.splitlines()) == 1 
355      yield "``%s``" % (name,) 
356      yield "  %s" % (doc,) 
357   
358   
359 -def _ManPageNodeClass(*args, **kwargs): 
360    """Generates a pending XRef like a ":doc:`...`" reference. 
361   
362    """ 
363    # Type for sphinx/environment.py:BuildEnvironment.resolve_references 
364    kwargs["reftype"] = "doc" 
365   
366    # Force custom title 
367    kwargs["refexplicit"] = True 
368   
369    return sphinx.addnodes.pending_xref(*args, **kwargs) 
370   
371   
372 -class _ManPageXRefRole(sphinx.roles.XRefRole): 
373 -  def __init__(self): 
374      """Initializes this class. 
375   
376      """ 
377      sphinx.roles.XRefRole.__init__(self, nodeclass=_ManPageNodeClass, 
378                                     warn_dangling=True) 
379   
380      assert not hasattr(self, "converted"), \ 
381        "Sphinx base class gained an attribute named 'converted'" 
382   
383      self.converted = None 
384   
385 -  def process_link(self, env, refnode, has_explicit_title, title, target): 
386      """Specialization for man page links. 
387   
388      """ 
389      if has_explicit_title: 
390        raise ReSTError("Setting explicit title is not allowed for man pages") 
391   
392      # Check format and extract name and section 
393      m = _MAN_RE.match(title) 
394      if not m: 
395        raise ReSTError("Man page reference '%s' does not match regular" 
396                        " expression '%s'" % (title, _MAN_RE.pattern)) 
397   
398      name = m.group("name") 
399      section = int(m.group("section")) 
400   
401      wanted_section = _constants.MAN_PAGES.get(name, None) 
402   
403      if not (wanted_section is None or wanted_section == section): 
404        raise ReSTError("Referenced man page '%s' has section number %s, but the" 
405                        " reference uses section %s" % 
406                        (name, wanted_section, section)) 
407   
408      self.converted = bool(wanted_section is not None and 
409                            env.app.config.enable_manpages) 
410   
411      if self.converted: 
412        # Create link to known man page 
413        return (title, "man-%s" % name) 
414      else: 
415        # No changes 
416        return (title, target) 
417   
418   
419 -def _ManPageRole(typ, rawtext, text, lineno, inliner, # pylint: disable=W0102 
420                   options={}, content=[]): 
421    """Custom role for man page references. 
422   
423    Converts man pages to links if enabled during the build. 
424   
425    """ 
426    xref = _ManPageXRefRole() 
427   
428    assert ht.TNone(xref.converted) 
429   
430    # Check if it's a known man page 
431    try: 
432      result = xref(typ, rawtext, text, lineno, inliner, 
433                    options=options, content=content) 
434    except ReSTError, err: 
435      msg = inliner.reporter.error(str(err), line=lineno) 
436      return ([inliner.problematic(rawtext, rawtext, msg)], [msg]) 
437   
438    assert ht.TBool(xref.converted) 
439   
440    # Return if the conversion was successful (i.e. the man page was known and 
441    # conversion was enabled) 
442    if xref.converted: 
443      return result 
444   
445    # Fallback if man page links are disabled or an unknown page is referenced 
446    return orig_manpage_role(typ, rawtext, text, lineno, inliner, 
447                             options=options, content=content) 
448   
449   
450 -def _EncodeRapiResourceLink(method, uri): 
451    """Encodes a RAPI resource URI for use as a link target. 
452   
453    """ 
454    parts = [RAPI_URI_ENCODE_RE.sub("-", uri.lower()).strip("-")] 
455   
456    if method is not None: 
457      parts.append(method.lower()) 
458   
459    return "rapi-res-%s" % "+".join(filter(None, parts)) 
460   
461   
462 -def _MakeRapiResourceLink(method, uri): 
463    """Generates link target name for RAPI resource. 
464   
465    """ 
466    if uri in ["/", "/2"]: 
467      # Don't link these 
468      return None 
469   
470    elif uri == "/version": 
471      return _EncodeRapiResourceLink(method, uri) 
472   
473    elif uri.startswith("/2/"): 
474      return _EncodeRapiResourceLink(method, uri[len("/2/"):]) 
475   
476    else: 
477      raise ReSTError("Unhandled URI '%s'" % uri) 
478   
479   
480 -def _GetHandlerMethods(handler): 
481    """Returns list of HTTP methods supported by handler class. 
482   
483    @type handler: L{rapi.baserlib.ResourceBase} 
484    @param handler: Handler class 
485    @rtype: list of strings 
486   
487    """ 
488    return sorted(method 
489                  for (method, op_attr, _, _, _) in rapi.baserlib.OPCODE_ATTRS 
490                  # Only if handler supports method 
491                  if hasattr(handler, method) or hasattr(handler, op_attr)) 
492   
493   
494 -def _DescribeHandlerAccess(handler, method): 
495    """Returns textual description of required RAPI permissions. 
496   
497    @type handler: L{rapi.baserlib.ResourceBase} 
498    @param handler: Handler class 
499    @type method: string 
500    @param method: HTTP method (e.g. L{http.HTTP_GET}) 
501    @rtype: string 
502   
503    """ 
504    access = rapi.baserlib.GetHandlerAccess(handler, method) 
505   
506    if access: 
507      return utils.CommaJoin(sorted(access)) 
508    else: 
509      return "*(none)*" 
510   
511   
512 -class _RapiHandlersForDocsHelper(object): 
513    @classmethod 
514 -  def Build(cls): 
515      """Returns dictionary of resource handlers. 
516   
517      """ 
518      resources = \ 
519        rapi.connector.GetHandlers("[node_name]", "[instance_name]", 
520                                   "[group_name]", "[network_name]", "[job_id]", 
521                                   "[disk_index]", "[resource]", 
522                                   translate=cls._TranslateResourceUri) 
523   
524      return resources 
525   
526    @classmethod 
527 -  def _TranslateResourceUri(cls, *args): 
528      """Translates a resource URI for use in documentation. 
529   
530      @see: L{rapi.connector.GetHandlers} 
531   
532      """ 
533      return "".join(map(cls._UriPatternToString, args)) 
534   
535    @staticmethod 
536 -  def _UriPatternToString(value): 
537      """Converts L{rapi.connector.UriPattern} to strings. 
538   
539      """ 
540      if isinstance(value, rapi.connector.UriPattern): 
541        return value.content 
542      else: 
543        return value 
544   
545   
546  _RAPI_RESOURCES_FOR_DOCS = _RapiHandlersForDocsHelper.Build() 
547   
548   
549 -def _BuildRapiAccessTable(res): 
550    """Build a table with access permissions needed for all RAPI resources. 
551   
552    """ 
553    for (uri, handler) in utils.NiceSort(res.items(), key=compat.fst): 
554      reslink = _MakeRapiResourceLink(None, uri) 
555      if not reslink: 
556        # No link was generated 
557        continue 
558   
559      yield ":ref:`%s <%s>`" % (uri, reslink) 
560   
561      for method in _GetHandlerMethods(handler): 
562        yield ("  | :ref:`%s <%s>`: %s" % 
563               (method, _MakeRapiResourceLink(method, uri), 
564                _DescribeHandlerAccess(handler, method))) 
565   
566   
567 -class RapiAccessTable(s_compat.Directive): 
568    """Custom directive to generate table of all RAPI resources. 
569   
570    See also <http://docutils.sourceforge.net/docs/howto/rst-directives.html>. 
571   
572    """ 
573    has_content = False 
574    required_arguments = 0 
575    optional_arguments = 0 
576    final_argument_whitespace = False 
577    option_spec = {} 
578   
579 -  def run(self): 
580      include_text = "\n".join(_BuildRapiAccessTable(_RAPI_RESOURCES_FOR_DOCS)) 
581   
582      # Inject into state machine 
583      include_lines = docutils.statemachine.string2lines(include_text, _TAB_WIDTH, 
584                                                         convert_whitespace=1) 
585      self.state_machine.insert_input(include_lines, self.__class__.__name__) 
586   
587      return [] 
588   
589   
590 -class RapiResourceDetails(s_compat.Directive): 
591    """Custom directive for RAPI resource details. 
592   
593    See also <http://docutils.sourceforge.net/docs/howto/rst-directives.html>. 
594   
595    """ 
596    has_content = False 
597    required_arguments = 1 
598    optional_arguments = 0 
599    final_argument_whitespace = False 
600   
601 -  def run(self): 
602      uri = self.arguments[0] 
603   
604      try: 
605        handler = _RAPI_RESOURCES_FOR_DOCS[uri] 
606      except KeyError: 
607        raise self.error("Unknown resource URI '%s'" % uri) 
608   
609      lines = [ 
610        ".. list-table::", 
611        "   :widths: 1 4", 
612        "   :header-rows: 1", 
613        "", 
614        "   * - Method", 
615        "     - :ref:`Required permissions <rapi-users>`", 
616        ] 
617   
618      for method in _GetHandlerMethods(handler): 
619        lines.extend([ 
620          "   * - :ref:`%s <%s>`" % (method, _MakeRapiResourceLink(method, uri)), 
621          "     - %s" % _DescribeHandlerAccess(handler, method), 
622          ]) 
623   
624      # Inject into state machine 
625      include_lines = \ 
626        docutils.statemachine.string2lines("\n".join(lines), _TAB_WIDTH, 
627                                           convert_whitespace=1) 
628      self.state_machine.insert_input(include_lines, self.__class__.__name__) 
629   
630      return [] 
631   
632   
633 -def setup(app): 
634    """Sphinx extension callback. 
635   
636    """ 
637    # TODO: Implement Sphinx directive for query fields 
638    app.add_directive("opcode_params", OpcodeParams) 
639    app.add_directive("opcode_result", OpcodeResult) 
640    app.add_directive("pyassert", PythonAssert) 
641    app.add_role("pyeval", PythonEvalRole) 
642    app.add_directive("rapi_access_table", RapiAccessTable) 
643    app.add_directive("rapi_resource_details", RapiResourceDetails) 
644   
645    app.add_config_value("enable_manpages", False, True) 
646    app.add_role("manpage", _ManPageRole) 
647