qemu-e2k/scripts/qapi.py

2036 lines
68 KiB
Python
Raw Normal View History

#
# QAPI helper library
#
# Copyright IBM, Corp. 2011
# Copyright (c) 2013-2016 Red Hat Inc.
#
# Authors:
# Anthony Liguori <aliguori@us.ibm.com>
qapi.py: Restructure lexer and parser The parser has a rather unorthodox structure: Until EOF: Read a section: Generator function get_expr() yields one section after the other, as a string. An unindented, non-empty line that isn't a comment starts a new section. Lexing: Split section into a list of tokens (strings), with help of generator function tokenize(). Parsing: Parse the first expression from the list of tokens, with parse(), throw away any remaining tokens. In parse_schema(): record value of an enum, union or struct key (if any) in the appropriate global table, append expression to the list of expressions. Return list of expressions. Known issues: (1) Indentation is significant, unlike in real JSON. (2) Neither lexer nor parser have any idea of source positions. Error reporting is hard, let's go shopping. (3) The one error we bother to detect, we "report" via raise. (4) The lexer silently ignores invalid characters. (5) If everything in a section gets ignored, the parser crashes. (6) The lexer treats a string containing a structural character exactly like the structural character. (7) Tokens trailing the first expression in a section are silently ignored. (8) The parser accepts any token in place of a colon. (9) The parser treats comma as optional. (10) parse() crashes on unexpected EOF. (11) parse_schema() crashes when a section's expression isn't a JSON object. Replace this piece of original art by a thoroughly unoriginal design. Takes care of (1), (2), (5), (6) and (7), and lays the groundwork for addressing the others. Generated source files remain unchanged. Signed-off-by: Markus Armbruster <armbru@redhat.com> Reviewed-by: Eric Blake <eblake@redhat.com> Message-id: 1374939721-7876-4-git-send-email-armbru@redhat.com Signed-off-by: Anthony Liguori <aliguori@us.ibm.com>
2013-07-27 17:41:55 +02:00
# Markus Armbruster <armbru@redhat.com>
#
# This work is licensed under the terms of the GNU GPL, version 2.
# See the COPYING file in the top-level directory.
import re
from ordereddict import OrderedDict
import errno
import getopt
import os
import sys
import string
builtin_types = {
'str': 'QTYPE_QSTRING',
'int': 'QTYPE_QINT',
'number': 'QTYPE_QFLOAT',
'bool': 'QTYPE_QBOOL',
'int8': 'QTYPE_QINT',
'int16': 'QTYPE_QINT',
'int32': 'QTYPE_QINT',
'int64': 'QTYPE_QINT',
'uint8': 'QTYPE_QINT',
'uint16': 'QTYPE_QINT',
'uint32': 'QTYPE_QINT',
'uint64': 'QTYPE_QINT',
'size': 'QTYPE_QINT',
'any': None, # any QType possible, actually
'QType': 'QTYPE_QSTRING',
}
# Whitelist of commands allowed to return a non-dictionary
returns_whitelist = [
# From QMP:
'human-monitor-command',
'qom-get',
'query-migrate-cache-size',
'query-tpm-models',
'query-tpm-types',
'ringbuf-read',
# From QGA:
'guest-file-open',
'guest-fsfreeze-freeze',
'guest-fsfreeze-freeze-list',
'guest-fsfreeze-status',
'guest-fsfreeze-thaw',
'guest-get-time',
'guest-set-vcpus',
'guest-sync',
'guest-sync-delimited',
]
# Whitelist of entities allowed to violate case conventions
case_whitelist = [
# From QMP:
'ACPISlotType', # DIMM, visible through query-acpi-ospm-status
'CpuInfoMIPS', # PC, visible through query-cpu
'CpuInfoTricore', # PC, visible through query-cpu
'QapiErrorClass', # all members, visible through errors
'UuidInfo', # UUID, visible through query-uuid
'X86CPURegister32', # all members, visible indirectly through qom-get
'q_obj_CpuInfo-base', # CPU, visible through query-cpu
]
enum_types = []
struct_types = []
union_types = []
events = []
all_names = {}
#
# Parsing the schema into expressions
#
def error_path(parent):
res = ""
while parent:
res = ("In file included from %s:%d:\n" % (parent['file'],
parent['line'])) + res
parent = parent['parent']
return res
class QAPIError(Exception):
def __init__(self, fname, line, col, incl_info, msg):
Exception.__init__(self)
self.fname = fname
self.line = line
self.col = col
self.info = incl_info
self.msg = msg
def __str__(self):
loc = "%s:%d" % (self.fname, self.line)
if self.col is not None:
loc += ":%s" % self.col
return error_path(self.info) + "%s: %s" % (loc, self.msg)
class QAPIParseError(QAPIError):
def __init__(self, parser, msg):
col = 1
for ch in parser.src[parser.line_pos:parser.pos]:
if ch == '\t':
col = (col + 7) % 8 + 1
else:
col += 1
QAPIError.__init__(self, parser.fname, parser.line, col,
parser.incl_info, msg)
class QAPISemError(QAPIError):
def __init__(self, info, msg):
QAPIError.__init__(self, info['file'], info['line'], None,
info['parent'], msg)
qapi: add qapi2texi script As the name suggests, the qapi2texi script converts JSON QAPI description into a texi file suitable for different target formats (info/man/txt/pdf/html...). It parses the following kind of blocks: Free-form: ## # = Section # == Subsection # # Some text foo with *emphasis* # 1. with a list # 2. like that # # And some code: # | $ echo foo # | -> do this # | <- get that # ## Symbol description: ## # @symbol: # # Symbol body ditto ergo sum. Foo bar # baz ding. # # @param1: the frob to frobnicate # @param2: #optional how hard to frobnicate # # Returns: the frobnicated frob. # If frob isn't frobnicatable, GenericError. # # Since: version # Notes: notes, comments can have # - itemized list # - like this # # Example: # # -> { "execute": "quit" } # <- { "return": {} } # ## That's roughly following the following EBNF grammar: api_comment = "##\n" comment "##\n" comment = freeform_comment | symbol_comment freeform_comment = { "# " text "\n" | "#\n" } symbol_comment = "# @" name ":\n" { member | tag_section | freeform_comment } member = "# @" name ':' [ text ] "\n" freeform_comment tag_section = "# " ( "Returns:", "Since:", "Note:", "Notes:", "Example:", "Examples:" ) [ text ] "\n" freeform_comment text = free text with markup Note that the grammar is ambiguous: a line "# @foo:\n" can be parsed both as freeform_comment and as symbol_comment. The actual parser recognizes symbol_comment. See docs/qapi-code-gen.txt for more details. Deficiencies and limitations: - the generated QMP documentation includes internal types - union type support is lacking - type information is lacking in generated documentation - doc comment error message positions are imprecise, they point to the beginning of the comment. - a few minor issues, all marked TODO/FIXME in the code Signed-off-by: Marc-André Lureau <marcandre.lureau@redhat.com> Message-Id: <20170113144135.5150-16-marcandre.lureau@redhat.com> Reviewed-by: Markus Armbruster <armbru@redhat.com> [test-qapi.py tweaked to avoid trailing empty lines in .out] Signed-off-by: Markus Armbruster <armbru@redhat.com>
2017-01-13 15:41:29 +01:00
class QAPIDoc(object):
class Section(object):
def __init__(self, name=None):
# optional section name (argument/member or section name)
self.name = name
# the list of lines for this section
self.content = []
def append(self, line):
self.content.append(line)
def __repr__(self):
return "\n".join(self.content).strip()
class ArgSection(Section):
pass
def __init__(self, parser, info):
# self.parser is used to report errors with QAPIParseError. The
# resulting error position depends on the state of the parser.
# It happens to be the beginning of the comment. More or less
# servicable, but action at a distance.
self.parser = parser
self.info = info
self.symbol = None
self.body = QAPIDoc.Section()
# dict mapping parameter name to ArgSection
self.args = OrderedDict()
# a list of Section
self.sections = []
# the current section
self.section = self.body
# associated expression (to be set by expression parser)
self.expr = None
def has_section(self, name):
"""Return True if we have a section with this name."""
for i in self.sections:
if i.name == name:
return True
return False
def append(self, line):
"""Parse a comment line and add it to the documentation."""
line = line[1:]
if not line:
self._append_freeform(line)
return
if line[0] != ' ':
raise QAPIParseError(self.parser, "Missing space after #")
line = line[1:]
# FIXME not nice: things like '# @foo:' and '# @foo: ' aren't
# recognized, and get silently treated as ordinary text
if self.symbol:
self._append_symbol_line(line)
elif not self.body.content and line.startswith("@"):
if not line.endswith(":"):
raise QAPIParseError(self.parser, "Line should end with :")
self.symbol = line[1:-1]
# FIXME invalid names other than the empty string aren't flagged
if not self.symbol:
raise QAPIParseError(self.parser, "Invalid name")
else:
self._append_freeform(line)
def _append_symbol_line(self, line):
name = line.split(' ', 1)[0]
if name.startswith("@") and name.endswith(":"):
line = line[len(name)+1:]
self._start_args_section(name[1:-1])
elif name in ("Returns:", "Since:",
# those are often singular or plural
"Note:", "Notes:",
"Example:", "Examples:",
"TODO:"):
line = line[len(name)+1:]
self._start_section(name[:-1])
self._append_freeform(line)
def _start_args_section(self, name):
# FIXME invalid names other than the empty string aren't flagged
if not name:
raise QAPIParseError(self.parser, "Invalid parameter name")
if name in self.args:
raise QAPIParseError(self.parser,
"'%s' parameter name duplicated" % name)
if self.sections:
raise QAPIParseError(self.parser,
"'@%s:' can't follow '%s' section"
% (name, self.sections[0].name))
self.section = QAPIDoc.ArgSection(name)
self.args[name] = self.section
def _start_section(self, name=""):
if name in ("Returns", "Since") and self.has_section(name):
raise QAPIParseError(self.parser,
"Duplicated '%s' section" % name)
self.section = QAPIDoc.Section(name)
self.sections.append(self.section)
def _append_freeform(self, line):
in_arg = isinstance(self.section, QAPIDoc.ArgSection)
if (in_arg and self.section.content
and not self.section.content[-1]
and line and not line[0].isspace()):
self._start_section()
if (in_arg or not self.section.name
or not self.section.name.startswith("Example")):
line = line.strip()
self.section.append(line)
class QAPISchemaParser(object):
qapi.py: Restructure lexer and parser The parser has a rather unorthodox structure: Until EOF: Read a section: Generator function get_expr() yields one section after the other, as a string. An unindented, non-empty line that isn't a comment starts a new section. Lexing: Split section into a list of tokens (strings), with help of generator function tokenize(). Parsing: Parse the first expression from the list of tokens, with parse(), throw away any remaining tokens. In parse_schema(): record value of an enum, union or struct key (if any) in the appropriate global table, append expression to the list of expressions. Return list of expressions. Known issues: (1) Indentation is significant, unlike in real JSON. (2) Neither lexer nor parser have any idea of source positions. Error reporting is hard, let's go shopping. (3) The one error we bother to detect, we "report" via raise. (4) The lexer silently ignores invalid characters. (5) If everything in a section gets ignored, the parser crashes. (6) The lexer treats a string containing a structural character exactly like the structural character. (7) Tokens trailing the first expression in a section are silently ignored. (8) The parser accepts any token in place of a colon. (9) The parser treats comma as optional. (10) parse() crashes on unexpected EOF. (11) parse_schema() crashes when a section's expression isn't a JSON object. Replace this piece of original art by a thoroughly unoriginal design. Takes care of (1), (2), (5), (6) and (7), and lays the groundwork for addressing the others. Generated source files remain unchanged. Signed-off-by: Markus Armbruster <armbru@redhat.com> Reviewed-by: Eric Blake <eblake@redhat.com> Message-id: 1374939721-7876-4-git-send-email-armbru@redhat.com Signed-off-by: Anthony Liguori <aliguori@us.ibm.com>
2013-07-27 17:41:55 +02:00
def __init__(self, fp, previously_included=[], incl_info=None):
abs_fname = os.path.abspath(fp.name)
fname = fp.name
self.fname = fname
previously_included.append(abs_fname)
self.incl_info = incl_info
qapi.py: Restructure lexer and parser The parser has a rather unorthodox structure: Until EOF: Read a section: Generator function get_expr() yields one section after the other, as a string. An unindented, non-empty line that isn't a comment starts a new section. Lexing: Split section into a list of tokens (strings), with help of generator function tokenize(). Parsing: Parse the first expression from the list of tokens, with parse(), throw away any remaining tokens. In parse_schema(): record value of an enum, union or struct key (if any) in the appropriate global table, append expression to the list of expressions. Return list of expressions. Known issues: (1) Indentation is significant, unlike in real JSON. (2) Neither lexer nor parser have any idea of source positions. Error reporting is hard, let's go shopping. (3) The one error we bother to detect, we "report" via raise. (4) The lexer silently ignores invalid characters. (5) If everything in a section gets ignored, the parser crashes. (6) The lexer treats a string containing a structural character exactly like the structural character. (7) Tokens trailing the first expression in a section are silently ignored. (8) The parser accepts any token in place of a colon. (9) The parser treats comma as optional. (10) parse() crashes on unexpected EOF. (11) parse_schema() crashes when a section's expression isn't a JSON object. Replace this piece of original art by a thoroughly unoriginal design. Takes care of (1), (2), (5), (6) and (7), and lays the groundwork for addressing the others. Generated source files remain unchanged. Signed-off-by: Markus Armbruster <armbru@redhat.com> Reviewed-by: Eric Blake <eblake@redhat.com> Message-id: 1374939721-7876-4-git-send-email-armbru@redhat.com Signed-off-by: Anthony Liguori <aliguori@us.ibm.com>
2013-07-27 17:41:55 +02:00
self.src = fp.read()
if self.src == '' or self.src[-1] != '\n':
self.src += '\n'
self.cursor = 0
self.line = 1
self.line_pos = 0
qapi.py: Restructure lexer and parser The parser has a rather unorthodox structure: Until EOF: Read a section: Generator function get_expr() yields one section after the other, as a string. An unindented, non-empty line that isn't a comment starts a new section. Lexing: Split section into a list of tokens (strings), with help of generator function tokenize(). Parsing: Parse the first expression from the list of tokens, with parse(), throw away any remaining tokens. In parse_schema(): record value of an enum, union or struct key (if any) in the appropriate global table, append expression to the list of expressions. Return list of expressions. Known issues: (1) Indentation is significant, unlike in real JSON. (2) Neither lexer nor parser have any idea of source positions. Error reporting is hard, let's go shopping. (3) The one error we bother to detect, we "report" via raise. (4) The lexer silently ignores invalid characters. (5) If everything in a section gets ignored, the parser crashes. (6) The lexer treats a string containing a structural character exactly like the structural character. (7) Tokens trailing the first expression in a section are silently ignored. (8) The parser accepts any token in place of a colon. (9) The parser treats comma as optional. (10) parse() crashes on unexpected EOF. (11) parse_schema() crashes when a section's expression isn't a JSON object. Replace this piece of original art by a thoroughly unoriginal design. Takes care of (1), (2), (5), (6) and (7), and lays the groundwork for addressing the others. Generated source files remain unchanged. Signed-off-by: Markus Armbruster <armbru@redhat.com> Reviewed-by: Eric Blake <eblake@redhat.com> Message-id: 1374939721-7876-4-git-send-email-armbru@redhat.com Signed-off-by: Anthony Liguori <aliguori@us.ibm.com>
2013-07-27 17:41:55 +02:00
self.exprs = []
qapi: add qapi2texi script As the name suggests, the qapi2texi script converts JSON QAPI description into a texi file suitable for different target formats (info/man/txt/pdf/html...). It parses the following kind of blocks: Free-form: ## # = Section # == Subsection # # Some text foo with *emphasis* # 1. with a list # 2. like that # # And some code: # | $ echo foo # | -> do this # | <- get that # ## Symbol description: ## # @symbol: # # Symbol body ditto ergo sum. Foo bar # baz ding. # # @param1: the frob to frobnicate # @param2: #optional how hard to frobnicate # # Returns: the frobnicated frob. # If frob isn't frobnicatable, GenericError. # # Since: version # Notes: notes, comments can have # - itemized list # - like this # # Example: # # -> { "execute": "quit" } # <- { "return": {} } # ## That's roughly following the following EBNF grammar: api_comment = "##\n" comment "##\n" comment = freeform_comment | symbol_comment freeform_comment = { "# " text "\n" | "#\n" } symbol_comment = "# @" name ":\n" { member | tag_section | freeform_comment } member = "# @" name ':' [ text ] "\n" freeform_comment tag_section = "# " ( "Returns:", "Since:", "Note:", "Notes:", "Example:", "Examples:" ) [ text ] "\n" freeform_comment text = free text with markup Note that the grammar is ambiguous: a line "# @foo:\n" can be parsed both as freeform_comment and as symbol_comment. The actual parser recognizes symbol_comment. See docs/qapi-code-gen.txt for more details. Deficiencies and limitations: - the generated QMP documentation includes internal types - union type support is lacking - type information is lacking in generated documentation - doc comment error message positions are imprecise, they point to the beginning of the comment. - a few minor issues, all marked TODO/FIXME in the code Signed-off-by: Marc-André Lureau <marcandre.lureau@redhat.com> Message-Id: <20170113144135.5150-16-marcandre.lureau@redhat.com> Reviewed-by: Markus Armbruster <armbru@redhat.com> [test-qapi.py tweaked to avoid trailing empty lines in .out] Signed-off-by: Markus Armbruster <armbru@redhat.com>
2017-01-13 15:41:29 +01:00
self.docs = []
qapi.py: Restructure lexer and parser The parser has a rather unorthodox structure: Until EOF: Read a section: Generator function get_expr() yields one section after the other, as a string. An unindented, non-empty line that isn't a comment starts a new section. Lexing: Split section into a list of tokens (strings), with help of generator function tokenize(). Parsing: Parse the first expression from the list of tokens, with parse(), throw away any remaining tokens. In parse_schema(): record value of an enum, union or struct key (if any) in the appropriate global table, append expression to the list of expressions. Return list of expressions. Known issues: (1) Indentation is significant, unlike in real JSON. (2) Neither lexer nor parser have any idea of source positions. Error reporting is hard, let's go shopping. (3) The one error we bother to detect, we "report" via raise. (4) The lexer silently ignores invalid characters. (5) If everything in a section gets ignored, the parser crashes. (6) The lexer treats a string containing a structural character exactly like the structural character. (7) Tokens trailing the first expression in a section are silently ignored. (8) The parser accepts any token in place of a colon. (9) The parser treats comma as optional. (10) parse() crashes on unexpected EOF. (11) parse_schema() crashes when a section's expression isn't a JSON object. Replace this piece of original art by a thoroughly unoriginal design. Takes care of (1), (2), (5), (6) and (7), and lays the groundwork for addressing the others. Generated source files remain unchanged. Signed-off-by: Markus Armbruster <armbru@redhat.com> Reviewed-by: Eric Blake <eblake@redhat.com> Message-id: 1374939721-7876-4-git-send-email-armbru@redhat.com Signed-off-by: Anthony Liguori <aliguori@us.ibm.com>
2013-07-27 17:41:55 +02:00
self.accept()
while self.tok is not None:
info = {'file': fname, 'line': self.line,
'parent': self.incl_info}
qapi: add qapi2texi script As the name suggests, the qapi2texi script converts JSON QAPI description into a texi file suitable for different target formats (info/man/txt/pdf/html...). It parses the following kind of blocks: Free-form: ## # = Section # == Subsection # # Some text foo with *emphasis* # 1. with a list # 2. like that # # And some code: # | $ echo foo # | -> do this # | <- get that # ## Symbol description: ## # @symbol: # # Symbol body ditto ergo sum. Foo bar # baz ding. # # @param1: the frob to frobnicate # @param2: #optional how hard to frobnicate # # Returns: the frobnicated frob. # If frob isn't frobnicatable, GenericError. # # Since: version # Notes: notes, comments can have # - itemized list # - like this # # Example: # # -> { "execute": "quit" } # <- { "return": {} } # ## That's roughly following the following EBNF grammar: api_comment = "##\n" comment "##\n" comment = freeform_comment | symbol_comment freeform_comment = { "# " text "\n" | "#\n" } symbol_comment = "# @" name ":\n" { member | tag_section | freeform_comment } member = "# @" name ':' [ text ] "\n" freeform_comment tag_section = "# " ( "Returns:", "Since:", "Note:", "Notes:", "Example:", "Examples:" ) [ text ] "\n" freeform_comment text = free text with markup Note that the grammar is ambiguous: a line "# @foo:\n" can be parsed both as freeform_comment and as symbol_comment. The actual parser recognizes symbol_comment. See docs/qapi-code-gen.txt for more details. Deficiencies and limitations: - the generated QMP documentation includes internal types - union type support is lacking - type information is lacking in generated documentation - doc comment error message positions are imprecise, they point to the beginning of the comment. - a few minor issues, all marked TODO/FIXME in the code Signed-off-by: Marc-André Lureau <marcandre.lureau@redhat.com> Message-Id: <20170113144135.5150-16-marcandre.lureau@redhat.com> Reviewed-by: Markus Armbruster <armbru@redhat.com> [test-qapi.py tweaked to avoid trailing empty lines in .out] Signed-off-by: Markus Armbruster <armbru@redhat.com>
2017-01-13 15:41:29 +01:00
if self.tok == '#':
doc = self.get_doc(info)
self.docs.append(doc)
continue
expr = self.get_expr(False)
if isinstance(expr, dict) and "include" in expr:
if len(expr) != 1:
raise QAPISemError(info, "Invalid 'include' directive")
include = expr["include"]
if not isinstance(include, str):
raise QAPISemError(info,
"Value of 'include' must be a string")
incl_abs_fname = os.path.join(os.path.dirname(abs_fname),
include)
# catch inclusion cycle
inf = info
while inf:
if incl_abs_fname == os.path.abspath(inf['file']):
raise QAPISemError(info, "Inclusion loop for %s"
% include)
inf = inf['parent']
qapi: add qapi2texi script As the name suggests, the qapi2texi script converts JSON QAPI description into a texi file suitable for different target formats (info/man/txt/pdf/html...). It parses the following kind of blocks: Free-form: ## # = Section # == Subsection # # Some text foo with *emphasis* # 1. with a list # 2. like that # # And some code: # | $ echo foo # | -> do this # | <- get that # ## Symbol description: ## # @symbol: # # Symbol body ditto ergo sum. Foo bar # baz ding. # # @param1: the frob to frobnicate # @param2: #optional how hard to frobnicate # # Returns: the frobnicated frob. # If frob isn't frobnicatable, GenericError. # # Since: version # Notes: notes, comments can have # - itemized list # - like this # # Example: # # -> { "execute": "quit" } # <- { "return": {} } # ## That's roughly following the following EBNF grammar: api_comment = "##\n" comment "##\n" comment = freeform_comment | symbol_comment freeform_comment = { "# " text "\n" | "#\n" } symbol_comment = "# @" name ":\n" { member | tag_section | freeform_comment } member = "# @" name ':' [ text ] "\n" freeform_comment tag_section = "# " ( "Returns:", "Since:", "Note:", "Notes:", "Example:", "Examples:" ) [ text ] "\n" freeform_comment text = free text with markup Note that the grammar is ambiguous: a line "# @foo:\n" can be parsed both as freeform_comment and as symbol_comment. The actual parser recognizes symbol_comment. See docs/qapi-code-gen.txt for more details. Deficiencies and limitations: - the generated QMP documentation includes internal types - union type support is lacking - type information is lacking in generated documentation - doc comment error message positions are imprecise, they point to the beginning of the comment. - a few minor issues, all marked TODO/FIXME in the code Signed-off-by: Marc-André Lureau <marcandre.lureau@redhat.com> Message-Id: <20170113144135.5150-16-marcandre.lureau@redhat.com> Reviewed-by: Markus Armbruster <armbru@redhat.com> [test-qapi.py tweaked to avoid trailing empty lines in .out] Signed-off-by: Markus Armbruster <armbru@redhat.com>
2017-01-13 15:41:29 +01:00
# skip multiple include of the same file
if incl_abs_fname in previously_included:
continue
try:
fobj = open(incl_abs_fname, 'r')
except IOError as e:
raise QAPISemError(info, '%s: %s' % (e.strerror, include))
exprs_include = QAPISchemaParser(fobj, previously_included,
info)
self.exprs.extend(exprs_include.exprs)
qapi: add qapi2texi script As the name suggests, the qapi2texi script converts JSON QAPI description into a texi file suitable for different target formats (info/man/txt/pdf/html...). It parses the following kind of blocks: Free-form: ## # = Section # == Subsection # # Some text foo with *emphasis* # 1. with a list # 2. like that # # And some code: # | $ echo foo # | -> do this # | <- get that # ## Symbol description: ## # @symbol: # # Symbol body ditto ergo sum. Foo bar # baz ding. # # @param1: the frob to frobnicate # @param2: #optional how hard to frobnicate # # Returns: the frobnicated frob. # If frob isn't frobnicatable, GenericError. # # Since: version # Notes: notes, comments can have # - itemized list # - like this # # Example: # # -> { "execute": "quit" } # <- { "return": {} } # ## That's roughly following the following EBNF grammar: api_comment = "##\n" comment "##\n" comment = freeform_comment | symbol_comment freeform_comment = { "# " text "\n" | "#\n" } symbol_comment = "# @" name ":\n" { member | tag_section | freeform_comment } member = "# @" name ':' [ text ] "\n" freeform_comment tag_section = "# " ( "Returns:", "Since:", "Note:", "Notes:", "Example:", "Examples:" ) [ text ] "\n" freeform_comment text = free text with markup Note that the grammar is ambiguous: a line "# @foo:\n" can be parsed both as freeform_comment and as symbol_comment. The actual parser recognizes symbol_comment. See docs/qapi-code-gen.txt for more details. Deficiencies and limitations: - the generated QMP documentation includes internal types - union type support is lacking - type information is lacking in generated documentation - doc comment error message positions are imprecise, they point to the beginning of the comment. - a few minor issues, all marked TODO/FIXME in the code Signed-off-by: Marc-André Lureau <marcandre.lureau@redhat.com> Message-Id: <20170113144135.5150-16-marcandre.lureau@redhat.com> Reviewed-by: Markus Armbruster <armbru@redhat.com> [test-qapi.py tweaked to avoid trailing empty lines in .out] Signed-off-by: Markus Armbruster <armbru@redhat.com>
2017-01-13 15:41:29 +01:00
self.docs.extend(exprs_include.docs)
else:
expr_elem = {'expr': expr,
'info': info}
qapi: add qapi2texi script As the name suggests, the qapi2texi script converts JSON QAPI description into a texi file suitable for different target formats (info/man/txt/pdf/html...). It parses the following kind of blocks: Free-form: ## # = Section # == Subsection # # Some text foo with *emphasis* # 1. with a list # 2. like that # # And some code: # | $ echo foo # | -> do this # | <- get that # ## Symbol description: ## # @symbol: # # Symbol body ditto ergo sum. Foo bar # baz ding. # # @param1: the frob to frobnicate # @param2: #optional how hard to frobnicate # # Returns: the frobnicated frob. # If frob isn't frobnicatable, GenericError. # # Since: version # Notes: notes, comments can have # - itemized list # - like this # # Example: # # -> { "execute": "quit" } # <- { "return": {} } # ## That's roughly following the following EBNF grammar: api_comment = "##\n" comment "##\n" comment = freeform_comment | symbol_comment freeform_comment = { "# " text "\n" | "#\n" } symbol_comment = "# @" name ":\n" { member | tag_section | freeform_comment } member = "# @" name ':' [ text ] "\n" freeform_comment tag_section = "# " ( "Returns:", "Since:", "Note:", "Notes:", "Example:", "Examples:" ) [ text ] "\n" freeform_comment text = free text with markup Note that the grammar is ambiguous: a line "# @foo:\n" can be parsed both as freeform_comment and as symbol_comment. The actual parser recognizes symbol_comment. See docs/qapi-code-gen.txt for more details. Deficiencies and limitations: - the generated QMP documentation includes internal types - union type support is lacking - type information is lacking in generated documentation - doc comment error message positions are imprecise, they point to the beginning of the comment. - a few minor issues, all marked TODO/FIXME in the code Signed-off-by: Marc-André Lureau <marcandre.lureau@redhat.com> Message-Id: <20170113144135.5150-16-marcandre.lureau@redhat.com> Reviewed-by: Markus Armbruster <armbru@redhat.com> [test-qapi.py tweaked to avoid trailing empty lines in .out] Signed-off-by: Markus Armbruster <armbru@redhat.com>
2017-01-13 15:41:29 +01:00
if (self.docs
and self.docs[-1].info['file'] == fname
and not self.docs[-1].expr):
self.docs[-1].expr = expr
expr_elem['doc'] = self.docs[-1]
self.exprs.append(expr_elem)
qapi.py: Restructure lexer and parser The parser has a rather unorthodox structure: Until EOF: Read a section: Generator function get_expr() yields one section after the other, as a string. An unindented, non-empty line that isn't a comment starts a new section. Lexing: Split section into a list of tokens (strings), with help of generator function tokenize(). Parsing: Parse the first expression from the list of tokens, with parse(), throw away any remaining tokens. In parse_schema(): record value of an enum, union or struct key (if any) in the appropriate global table, append expression to the list of expressions. Return list of expressions. Known issues: (1) Indentation is significant, unlike in real JSON. (2) Neither lexer nor parser have any idea of source positions. Error reporting is hard, let's go shopping. (3) The one error we bother to detect, we "report" via raise. (4) The lexer silently ignores invalid characters. (5) If everything in a section gets ignored, the parser crashes. (6) The lexer treats a string containing a structural character exactly like the structural character. (7) Tokens trailing the first expression in a section are silently ignored. (8) The parser accepts any token in place of a colon. (9) The parser treats comma as optional. (10) parse() crashes on unexpected EOF. (11) parse_schema() crashes when a section's expression isn't a JSON object. Replace this piece of original art by a thoroughly unoriginal design. Takes care of (1), (2), (5), (6) and (7), and lays the groundwork for addressing the others. Generated source files remain unchanged. Signed-off-by: Markus Armbruster <armbru@redhat.com> Reviewed-by: Eric Blake <eblake@redhat.com> Message-id: 1374939721-7876-4-git-send-email-armbru@redhat.com Signed-off-by: Anthony Liguori <aliguori@us.ibm.com>
2013-07-27 17:41:55 +02:00
qapi: add qapi2texi script As the name suggests, the qapi2texi script converts JSON QAPI description into a texi file suitable for different target formats (info/man/txt/pdf/html...). It parses the following kind of blocks: Free-form: ## # = Section # == Subsection # # Some text foo with *emphasis* # 1. with a list # 2. like that # # And some code: # | $ echo foo # | -> do this # | <- get that # ## Symbol description: ## # @symbol: # # Symbol body ditto ergo sum. Foo bar # baz ding. # # @param1: the frob to frobnicate # @param2: #optional how hard to frobnicate # # Returns: the frobnicated frob. # If frob isn't frobnicatable, GenericError. # # Since: version # Notes: notes, comments can have # - itemized list # - like this # # Example: # # -> { "execute": "quit" } # <- { "return": {} } # ## That's roughly following the following EBNF grammar: api_comment = "##\n" comment "##\n" comment = freeform_comment | symbol_comment freeform_comment = { "# " text "\n" | "#\n" } symbol_comment = "# @" name ":\n" { member | tag_section | freeform_comment } member = "# @" name ':' [ text ] "\n" freeform_comment tag_section = "# " ( "Returns:", "Since:", "Note:", "Notes:", "Example:", "Examples:" ) [ text ] "\n" freeform_comment text = free text with markup Note that the grammar is ambiguous: a line "# @foo:\n" can be parsed both as freeform_comment and as symbol_comment. The actual parser recognizes symbol_comment. See docs/qapi-code-gen.txt for more details. Deficiencies and limitations: - the generated QMP documentation includes internal types - union type support is lacking - type information is lacking in generated documentation - doc comment error message positions are imprecise, they point to the beginning of the comment. - a few minor issues, all marked TODO/FIXME in the code Signed-off-by: Marc-André Lureau <marcandre.lureau@redhat.com> Message-Id: <20170113144135.5150-16-marcandre.lureau@redhat.com> Reviewed-by: Markus Armbruster <armbru@redhat.com> [test-qapi.py tweaked to avoid trailing empty lines in .out] Signed-off-by: Markus Armbruster <armbru@redhat.com>
2017-01-13 15:41:29 +01:00
def accept(self, skip_comment=True):
qapi.py: Restructure lexer and parser The parser has a rather unorthodox structure: Until EOF: Read a section: Generator function get_expr() yields one section after the other, as a string. An unindented, non-empty line that isn't a comment starts a new section. Lexing: Split section into a list of tokens (strings), with help of generator function tokenize(). Parsing: Parse the first expression from the list of tokens, with parse(), throw away any remaining tokens. In parse_schema(): record value of an enum, union or struct key (if any) in the appropriate global table, append expression to the list of expressions. Return list of expressions. Known issues: (1) Indentation is significant, unlike in real JSON. (2) Neither lexer nor parser have any idea of source positions. Error reporting is hard, let's go shopping. (3) The one error we bother to detect, we "report" via raise. (4) The lexer silently ignores invalid characters. (5) If everything in a section gets ignored, the parser crashes. (6) The lexer treats a string containing a structural character exactly like the structural character. (7) Tokens trailing the first expression in a section are silently ignored. (8) The parser accepts any token in place of a colon. (9) The parser treats comma as optional. (10) parse() crashes on unexpected EOF. (11) parse_schema() crashes when a section's expression isn't a JSON object. Replace this piece of original art by a thoroughly unoriginal design. Takes care of (1), (2), (5), (6) and (7), and lays the groundwork for addressing the others. Generated source files remain unchanged. Signed-off-by: Markus Armbruster <armbru@redhat.com> Reviewed-by: Eric Blake <eblake@redhat.com> Message-id: 1374939721-7876-4-git-send-email-armbru@redhat.com Signed-off-by: Anthony Liguori <aliguori@us.ibm.com>
2013-07-27 17:41:55 +02:00
while True:
self.tok = self.src[self.cursor]
self.pos = self.cursor
qapi.py: Restructure lexer and parser The parser has a rather unorthodox structure: Until EOF: Read a section: Generator function get_expr() yields one section after the other, as a string. An unindented, non-empty line that isn't a comment starts a new section. Lexing: Split section into a list of tokens (strings), with help of generator function tokenize(). Parsing: Parse the first expression from the list of tokens, with parse(), throw away any remaining tokens. In parse_schema(): record value of an enum, union or struct key (if any) in the appropriate global table, append expression to the list of expressions. Return list of expressions. Known issues: (1) Indentation is significant, unlike in real JSON. (2) Neither lexer nor parser have any idea of source positions. Error reporting is hard, let's go shopping. (3) The one error we bother to detect, we "report" via raise. (4) The lexer silently ignores invalid characters. (5) If everything in a section gets ignored, the parser crashes. (6) The lexer treats a string containing a structural character exactly like the structural character. (7) Tokens trailing the first expression in a section are silently ignored. (8) The parser accepts any token in place of a colon. (9) The parser treats comma as optional. (10) parse() crashes on unexpected EOF. (11) parse_schema() crashes when a section's expression isn't a JSON object. Replace this piece of original art by a thoroughly unoriginal design. Takes care of (1), (2), (5), (6) and (7), and lays the groundwork for addressing the others. Generated source files remain unchanged. Signed-off-by: Markus Armbruster <armbru@redhat.com> Reviewed-by: Eric Blake <eblake@redhat.com> Message-id: 1374939721-7876-4-git-send-email-armbru@redhat.com Signed-off-by: Anthony Liguori <aliguori@us.ibm.com>
2013-07-27 17:41:55 +02:00
self.cursor += 1
self.val = None
if self.tok == '#':
qapi: add qapi2texi script As the name suggests, the qapi2texi script converts JSON QAPI description into a texi file suitable for different target formats (info/man/txt/pdf/html...). It parses the following kind of blocks: Free-form: ## # = Section # == Subsection # # Some text foo with *emphasis* # 1. with a list # 2. like that # # And some code: # | $ echo foo # | -> do this # | <- get that # ## Symbol description: ## # @symbol: # # Symbol body ditto ergo sum. Foo bar # baz ding. # # @param1: the frob to frobnicate # @param2: #optional how hard to frobnicate # # Returns: the frobnicated frob. # If frob isn't frobnicatable, GenericError. # # Since: version # Notes: notes, comments can have # - itemized list # - like this # # Example: # # -> { "execute": "quit" } # <- { "return": {} } # ## That's roughly following the following EBNF grammar: api_comment = "##\n" comment "##\n" comment = freeform_comment | symbol_comment freeform_comment = { "# " text "\n" | "#\n" } symbol_comment = "# @" name ":\n" { member | tag_section | freeform_comment } member = "# @" name ':' [ text ] "\n" freeform_comment tag_section = "# " ( "Returns:", "Since:", "Note:", "Notes:", "Example:", "Examples:" ) [ text ] "\n" freeform_comment text = free text with markup Note that the grammar is ambiguous: a line "# @foo:\n" can be parsed both as freeform_comment and as symbol_comment. The actual parser recognizes symbol_comment. See docs/qapi-code-gen.txt for more details. Deficiencies and limitations: - the generated QMP documentation includes internal types - union type support is lacking - type information is lacking in generated documentation - doc comment error message positions are imprecise, they point to the beginning of the comment. - a few minor issues, all marked TODO/FIXME in the code Signed-off-by: Marc-André Lureau <marcandre.lureau@redhat.com> Message-Id: <20170113144135.5150-16-marcandre.lureau@redhat.com> Reviewed-by: Markus Armbruster <armbru@redhat.com> [test-qapi.py tweaked to avoid trailing empty lines in .out] Signed-off-by: Markus Armbruster <armbru@redhat.com>
2017-01-13 15:41:29 +01:00
if self.src[self.cursor] == '#':
# Start of doc comment
skip_comment = False
qapi.py: Restructure lexer and parser The parser has a rather unorthodox structure: Until EOF: Read a section: Generator function get_expr() yields one section after the other, as a string. An unindented, non-empty line that isn't a comment starts a new section. Lexing: Split section into a list of tokens (strings), with help of generator function tokenize(). Parsing: Parse the first expression from the list of tokens, with parse(), throw away any remaining tokens. In parse_schema(): record value of an enum, union or struct key (if any) in the appropriate global table, append expression to the list of expressions. Return list of expressions. Known issues: (1) Indentation is significant, unlike in real JSON. (2) Neither lexer nor parser have any idea of source positions. Error reporting is hard, let's go shopping. (3) The one error we bother to detect, we "report" via raise. (4) The lexer silently ignores invalid characters. (5) If everything in a section gets ignored, the parser crashes. (6) The lexer treats a string containing a structural character exactly like the structural character. (7) Tokens trailing the first expression in a section are silently ignored. (8) The parser accepts any token in place of a colon. (9) The parser treats comma as optional. (10) parse() crashes on unexpected EOF. (11) parse_schema() crashes when a section's expression isn't a JSON object. Replace this piece of original art by a thoroughly unoriginal design. Takes care of (1), (2), (5), (6) and (7), and lays the groundwork for addressing the others. Generated source files remain unchanged. Signed-off-by: Markus Armbruster <armbru@redhat.com> Reviewed-by: Eric Blake <eblake@redhat.com> Message-id: 1374939721-7876-4-git-send-email-armbru@redhat.com Signed-off-by: Anthony Liguori <aliguori@us.ibm.com>
2013-07-27 17:41:55 +02:00
self.cursor = self.src.find('\n', self.cursor)
qapi: add qapi2texi script As the name suggests, the qapi2texi script converts JSON QAPI description into a texi file suitable for different target formats (info/man/txt/pdf/html...). It parses the following kind of blocks: Free-form: ## # = Section # == Subsection # # Some text foo with *emphasis* # 1. with a list # 2. like that # # And some code: # | $ echo foo # | -> do this # | <- get that # ## Symbol description: ## # @symbol: # # Symbol body ditto ergo sum. Foo bar # baz ding. # # @param1: the frob to frobnicate # @param2: #optional how hard to frobnicate # # Returns: the frobnicated frob. # If frob isn't frobnicatable, GenericError. # # Since: version # Notes: notes, comments can have # - itemized list # - like this # # Example: # # -> { "execute": "quit" } # <- { "return": {} } # ## That's roughly following the following EBNF grammar: api_comment = "##\n" comment "##\n" comment = freeform_comment | symbol_comment freeform_comment = { "# " text "\n" | "#\n" } symbol_comment = "# @" name ":\n" { member | tag_section | freeform_comment } member = "# @" name ':' [ text ] "\n" freeform_comment tag_section = "# " ( "Returns:", "Since:", "Note:", "Notes:", "Example:", "Examples:" ) [ text ] "\n" freeform_comment text = free text with markup Note that the grammar is ambiguous: a line "# @foo:\n" can be parsed both as freeform_comment and as symbol_comment. The actual parser recognizes symbol_comment. See docs/qapi-code-gen.txt for more details. Deficiencies and limitations: - the generated QMP documentation includes internal types - union type support is lacking - type information is lacking in generated documentation - doc comment error message positions are imprecise, they point to the beginning of the comment. - a few minor issues, all marked TODO/FIXME in the code Signed-off-by: Marc-André Lureau <marcandre.lureau@redhat.com> Message-Id: <20170113144135.5150-16-marcandre.lureau@redhat.com> Reviewed-by: Markus Armbruster <armbru@redhat.com> [test-qapi.py tweaked to avoid trailing empty lines in .out] Signed-off-by: Markus Armbruster <armbru@redhat.com>
2017-01-13 15:41:29 +01:00
if not skip_comment:
self.val = self.src[self.pos:self.cursor]
return
elif self.tok in "{}:,[]":
qapi.py: Restructure lexer and parser The parser has a rather unorthodox structure: Until EOF: Read a section: Generator function get_expr() yields one section after the other, as a string. An unindented, non-empty line that isn't a comment starts a new section. Lexing: Split section into a list of tokens (strings), with help of generator function tokenize(). Parsing: Parse the first expression from the list of tokens, with parse(), throw away any remaining tokens. In parse_schema(): record value of an enum, union or struct key (if any) in the appropriate global table, append expression to the list of expressions. Return list of expressions. Known issues: (1) Indentation is significant, unlike in real JSON. (2) Neither lexer nor parser have any idea of source positions. Error reporting is hard, let's go shopping. (3) The one error we bother to detect, we "report" via raise. (4) The lexer silently ignores invalid characters. (5) If everything in a section gets ignored, the parser crashes. (6) The lexer treats a string containing a structural character exactly like the structural character. (7) Tokens trailing the first expression in a section are silently ignored. (8) The parser accepts any token in place of a colon. (9) The parser treats comma as optional. (10) parse() crashes on unexpected EOF. (11) parse_schema() crashes when a section's expression isn't a JSON object. Replace this piece of original art by a thoroughly unoriginal design. Takes care of (1), (2), (5), (6) and (7), and lays the groundwork for addressing the others. Generated source files remain unchanged. Signed-off-by: Markus Armbruster <armbru@redhat.com> Reviewed-by: Eric Blake <eblake@redhat.com> Message-id: 1374939721-7876-4-git-send-email-armbru@redhat.com Signed-off-by: Anthony Liguori <aliguori@us.ibm.com>
2013-07-27 17:41:55 +02:00
return
elif self.tok == "'":
string = ''
esc = False
while True:
ch = self.src[self.cursor]
self.cursor += 1
if ch == '\n':
raise QAPIParseError(self, 'Missing terminating "\'"')
qapi.py: Restructure lexer and parser The parser has a rather unorthodox structure: Until EOF: Read a section: Generator function get_expr() yields one section after the other, as a string. An unindented, non-empty line that isn't a comment starts a new section. Lexing: Split section into a list of tokens (strings), with help of generator function tokenize(). Parsing: Parse the first expression from the list of tokens, with parse(), throw away any remaining tokens. In parse_schema(): record value of an enum, union or struct key (if any) in the appropriate global table, append expression to the list of expressions. Return list of expressions. Known issues: (1) Indentation is significant, unlike in real JSON. (2) Neither lexer nor parser have any idea of source positions. Error reporting is hard, let's go shopping. (3) The one error we bother to detect, we "report" via raise. (4) The lexer silently ignores invalid characters. (5) If everything in a section gets ignored, the parser crashes. (6) The lexer treats a string containing a structural character exactly like the structural character. (7) Tokens trailing the first expression in a section are silently ignored. (8) The parser accepts any token in place of a colon. (9) The parser treats comma as optional. (10) parse() crashes on unexpected EOF. (11) parse_schema() crashes when a section's expression isn't a JSON object. Replace this piece of original art by a thoroughly unoriginal design. Takes care of (1), (2), (5), (6) and (7), and lays the groundwork for addressing the others. Generated source files remain unchanged. Signed-off-by: Markus Armbruster <armbru@redhat.com> Reviewed-by: Eric Blake <eblake@redhat.com> Message-id: 1374939721-7876-4-git-send-email-armbru@redhat.com Signed-off-by: Anthony Liguori <aliguori@us.ibm.com>
2013-07-27 17:41:55 +02:00
if esc:
if ch == 'b':
string += '\b'
elif ch == 'f':
string += '\f'
elif ch == 'n':
string += '\n'
elif ch == 'r':
string += '\r'
elif ch == 't':
string += '\t'
elif ch == 'u':
value = 0
for _ in range(0, 4):
ch = self.src[self.cursor]
self.cursor += 1
if ch not in "0123456789abcdefABCDEF":
raise QAPIParseError(self,
'\\u escape needs 4 '
'hex digits')
value = (value << 4) + int(ch, 16)
# If Python 2 and 3 didn't disagree so much on
# how to handle Unicode, then we could allow
# Unicode string defaults. But most of QAPI is
# ASCII-only, so we aren't losing much for now.
if not value or value > 0x7f:
raise QAPIParseError(self,
'For now, \\u escape '
'only supports non-zero '
'values up to \\u007f')
string += chr(value)
elif ch in "\\/'\"":
string += ch
else:
raise QAPIParseError(self,
"Unknown escape \\%s" % ch)
qapi.py: Restructure lexer and parser The parser has a rather unorthodox structure: Until EOF: Read a section: Generator function get_expr() yields one section after the other, as a string. An unindented, non-empty line that isn't a comment starts a new section. Lexing: Split section into a list of tokens (strings), with help of generator function tokenize(). Parsing: Parse the first expression from the list of tokens, with parse(), throw away any remaining tokens. In parse_schema(): record value of an enum, union or struct key (if any) in the appropriate global table, append expression to the list of expressions. Return list of expressions. Known issues: (1) Indentation is significant, unlike in real JSON. (2) Neither lexer nor parser have any idea of source positions. Error reporting is hard, let's go shopping. (3) The one error we bother to detect, we "report" via raise. (4) The lexer silently ignores invalid characters. (5) If everything in a section gets ignored, the parser crashes. (6) The lexer treats a string containing a structural character exactly like the structural character. (7) Tokens trailing the first expression in a section are silently ignored. (8) The parser accepts any token in place of a colon. (9) The parser treats comma as optional. (10) parse() crashes on unexpected EOF. (11) parse_schema() crashes when a section's expression isn't a JSON object. Replace this piece of original art by a thoroughly unoriginal design. Takes care of (1), (2), (5), (6) and (7), and lays the groundwork for addressing the others. Generated source files remain unchanged. Signed-off-by: Markus Armbruster <armbru@redhat.com> Reviewed-by: Eric Blake <eblake@redhat.com> Message-id: 1374939721-7876-4-git-send-email-armbru@redhat.com Signed-off-by: Anthony Liguori <aliguori@us.ibm.com>
2013-07-27 17:41:55 +02:00
esc = False
elif ch == "\\":
esc = True
elif ch == "'":
self.val = string
return
else:
string += ch
elif self.src.startswith("true", self.pos):
self.val = True
self.cursor += 3
return
elif self.src.startswith("false", self.pos):
self.val = False
self.cursor += 4
return
elif self.src.startswith("null", self.pos):
self.val = None
self.cursor += 3
return
qapi.py: Restructure lexer and parser The parser has a rather unorthodox structure: Until EOF: Read a section: Generator function get_expr() yields one section after the other, as a string. An unindented, non-empty line that isn't a comment starts a new section. Lexing: Split section into a list of tokens (strings), with help of generator function tokenize(). Parsing: Parse the first expression from the list of tokens, with parse(), throw away any remaining tokens. In parse_schema(): record value of an enum, union or struct key (if any) in the appropriate global table, append expression to the list of expressions. Return list of expressions. Known issues: (1) Indentation is significant, unlike in real JSON. (2) Neither lexer nor parser have any idea of source positions. Error reporting is hard, let's go shopping. (3) The one error we bother to detect, we "report" via raise. (4) The lexer silently ignores invalid characters. (5) If everything in a section gets ignored, the parser crashes. (6) The lexer treats a string containing a structural character exactly like the structural character. (7) Tokens trailing the first expression in a section are silently ignored. (8) The parser accepts any token in place of a colon. (9) The parser treats comma as optional. (10) parse() crashes on unexpected EOF. (11) parse_schema() crashes when a section's expression isn't a JSON object. Replace this piece of original art by a thoroughly unoriginal design. Takes care of (1), (2), (5), (6) and (7), and lays the groundwork for addressing the others. Generated source files remain unchanged. Signed-off-by: Markus Armbruster <armbru@redhat.com> Reviewed-by: Eric Blake <eblake@redhat.com> Message-id: 1374939721-7876-4-git-send-email-armbru@redhat.com Signed-off-by: Anthony Liguori <aliguori@us.ibm.com>
2013-07-27 17:41:55 +02:00
elif self.tok == '\n':
if self.cursor == len(self.src):
self.tok = None
return
self.line += 1
self.line_pos = self.cursor
elif not self.tok.isspace():
raise QAPIParseError(self, 'Stray "%s"' % self.tok)
qapi.py: Restructure lexer and parser The parser has a rather unorthodox structure: Until EOF: Read a section: Generator function get_expr() yields one section after the other, as a string. An unindented, non-empty line that isn't a comment starts a new section. Lexing: Split section into a list of tokens (strings), with help of generator function tokenize(). Parsing: Parse the first expression from the list of tokens, with parse(), throw away any remaining tokens. In parse_schema(): record value of an enum, union or struct key (if any) in the appropriate global table, append expression to the list of expressions. Return list of expressions. Known issues: (1) Indentation is significant, unlike in real JSON. (2) Neither lexer nor parser have any idea of source positions. Error reporting is hard, let's go shopping. (3) The one error we bother to detect, we "report" via raise. (4) The lexer silently ignores invalid characters. (5) If everything in a section gets ignored, the parser crashes. (6) The lexer treats a string containing a structural character exactly like the structural character. (7) Tokens trailing the first expression in a section are silently ignored. (8) The parser accepts any token in place of a colon. (9) The parser treats comma as optional. (10) parse() crashes on unexpected EOF. (11) parse_schema() crashes when a section's expression isn't a JSON object. Replace this piece of original art by a thoroughly unoriginal design. Takes care of (1), (2), (5), (6) and (7), and lays the groundwork for addressing the others. Generated source files remain unchanged. Signed-off-by: Markus Armbruster <armbru@redhat.com> Reviewed-by: Eric Blake <eblake@redhat.com> Message-id: 1374939721-7876-4-git-send-email-armbru@redhat.com Signed-off-by: Anthony Liguori <aliguori@us.ibm.com>
2013-07-27 17:41:55 +02:00
def get_members(self):
expr = OrderedDict()
if self.tok == '}':
self.accept()
return expr
if self.tok != "'":
raise QAPIParseError(self, 'Expected string or "}"')
while True:
qapi.py: Restructure lexer and parser The parser has a rather unorthodox structure: Until EOF: Read a section: Generator function get_expr() yields one section after the other, as a string. An unindented, non-empty line that isn't a comment starts a new section. Lexing: Split section into a list of tokens (strings), with help of generator function tokenize(). Parsing: Parse the first expression from the list of tokens, with parse(), throw away any remaining tokens. In parse_schema(): record value of an enum, union or struct key (if any) in the appropriate global table, append expression to the list of expressions. Return list of expressions. Known issues: (1) Indentation is significant, unlike in real JSON. (2) Neither lexer nor parser have any idea of source positions. Error reporting is hard, let's go shopping. (3) The one error we bother to detect, we "report" via raise. (4) The lexer silently ignores invalid characters. (5) If everything in a section gets ignored, the parser crashes. (6) The lexer treats a string containing a structural character exactly like the structural character. (7) Tokens trailing the first expression in a section are silently ignored. (8) The parser accepts any token in place of a colon. (9) The parser treats comma as optional. (10) parse() crashes on unexpected EOF. (11) parse_schema() crashes when a section's expression isn't a JSON object. Replace this piece of original art by a thoroughly unoriginal design. Takes care of (1), (2), (5), (6) and (7), and lays the groundwork for addressing the others. Generated source files remain unchanged. Signed-off-by: Markus Armbruster <armbru@redhat.com> Reviewed-by: Eric Blake <eblake@redhat.com> Message-id: 1374939721-7876-4-git-send-email-armbru@redhat.com Signed-off-by: Anthony Liguori <aliguori@us.ibm.com>
2013-07-27 17:41:55 +02:00
key = self.val
self.accept()
if self.tok != ':':
raise QAPIParseError(self, 'Expected ":"')
self.accept()
if key in expr:
raise QAPIParseError(self, 'Duplicate key "%s"' % key)
expr[key] = self.get_expr(True)
if self.tok == '}':
qapi.py: Restructure lexer and parser The parser has a rather unorthodox structure: Until EOF: Read a section: Generator function get_expr() yields one section after the other, as a string. An unindented, non-empty line that isn't a comment starts a new section. Lexing: Split section into a list of tokens (strings), with help of generator function tokenize(). Parsing: Parse the first expression from the list of tokens, with parse(), throw away any remaining tokens. In parse_schema(): record value of an enum, union or struct key (if any) in the appropriate global table, append expression to the list of expressions. Return list of expressions. Known issues: (1) Indentation is significant, unlike in real JSON. (2) Neither lexer nor parser have any idea of source positions. Error reporting is hard, let's go shopping. (3) The one error we bother to detect, we "report" via raise. (4) The lexer silently ignores invalid characters. (5) If everything in a section gets ignored, the parser crashes. (6) The lexer treats a string containing a structural character exactly like the structural character. (7) Tokens trailing the first expression in a section are silently ignored. (8) The parser accepts any token in place of a colon. (9) The parser treats comma as optional. (10) parse() crashes on unexpected EOF. (11) parse_schema() crashes when a section's expression isn't a JSON object. Replace this piece of original art by a thoroughly unoriginal design. Takes care of (1), (2), (5), (6) and (7), and lays the groundwork for addressing the others. Generated source files remain unchanged. Signed-off-by: Markus Armbruster <armbru@redhat.com> Reviewed-by: Eric Blake <eblake@redhat.com> Message-id: 1374939721-7876-4-git-send-email-armbru@redhat.com Signed-off-by: Anthony Liguori <aliguori@us.ibm.com>
2013-07-27 17:41:55 +02:00
self.accept()
return expr
if self.tok != ',':
raise QAPIParseError(self, 'Expected "," or "}"')
self.accept()
if self.tok != "'":
raise QAPIParseError(self, 'Expected string')
qapi.py: Restructure lexer and parser The parser has a rather unorthodox structure: Until EOF: Read a section: Generator function get_expr() yields one section after the other, as a string. An unindented, non-empty line that isn't a comment starts a new section. Lexing: Split section into a list of tokens (strings), with help of generator function tokenize(). Parsing: Parse the first expression from the list of tokens, with parse(), throw away any remaining tokens. In parse_schema(): record value of an enum, union or struct key (if any) in the appropriate global table, append expression to the list of expressions. Return list of expressions. Known issues: (1) Indentation is significant, unlike in real JSON. (2) Neither lexer nor parser have any idea of source positions. Error reporting is hard, let's go shopping. (3) The one error we bother to detect, we "report" via raise. (4) The lexer silently ignores invalid characters. (5) If everything in a section gets ignored, the parser crashes. (6) The lexer treats a string containing a structural character exactly like the structural character. (7) Tokens trailing the first expression in a section are silently ignored. (8) The parser accepts any token in place of a colon. (9) The parser treats comma as optional. (10) parse() crashes on unexpected EOF. (11) parse_schema() crashes when a section's expression isn't a JSON object. Replace this piece of original art by a thoroughly unoriginal design. Takes care of (1), (2), (5), (6) and (7), and lays the groundwork for addressing the others. Generated source files remain unchanged. Signed-off-by: Markus Armbruster <armbru@redhat.com> Reviewed-by: Eric Blake <eblake@redhat.com> Message-id: 1374939721-7876-4-git-send-email-armbru@redhat.com Signed-off-by: Anthony Liguori <aliguori@us.ibm.com>
2013-07-27 17:41:55 +02:00
def get_values(self):
expr = []
if self.tok == ']':
self.accept()
return expr
if self.tok not in "{['tfn":
raise QAPIParseError(self, 'Expected "{", "[", "]", string, '
'boolean or "null"')
while True:
expr.append(self.get_expr(True))
if self.tok == ']':
qapi.py: Restructure lexer and parser The parser has a rather unorthodox structure: Until EOF: Read a section: Generator function get_expr() yields one section after the other, as a string. An unindented, non-empty line that isn't a comment starts a new section. Lexing: Split section into a list of tokens (strings), with help of generator function tokenize(). Parsing: Parse the first expression from the list of tokens, with parse(), throw away any remaining tokens. In parse_schema(): record value of an enum, union or struct key (if any) in the appropriate global table, append expression to the list of expressions. Return list of expressions. Known issues: (1) Indentation is significant, unlike in real JSON. (2) Neither lexer nor parser have any idea of source positions. Error reporting is hard, let's go shopping. (3) The one error we bother to detect, we "report" via raise. (4) The lexer silently ignores invalid characters. (5) If everything in a section gets ignored, the parser crashes. (6) The lexer treats a string containing a structural character exactly like the structural character. (7) Tokens trailing the first expression in a section are silently ignored. (8) The parser accepts any token in place of a colon. (9) The parser treats comma as optional. (10) parse() crashes on unexpected EOF. (11) parse_schema() crashes when a section's expression isn't a JSON object. Replace this piece of original art by a thoroughly unoriginal design. Takes care of (1), (2), (5), (6) and (7), and lays the groundwork for addressing the others. Generated source files remain unchanged. Signed-off-by: Markus Armbruster <armbru@redhat.com> Reviewed-by: Eric Blake <eblake@redhat.com> Message-id: 1374939721-7876-4-git-send-email-armbru@redhat.com Signed-off-by: Anthony Liguori <aliguori@us.ibm.com>
2013-07-27 17:41:55 +02:00
self.accept()
return expr
if self.tok != ',':
raise QAPIParseError(self, 'Expected "," or "]"')
self.accept()
qapi.py: Restructure lexer and parser The parser has a rather unorthodox structure: Until EOF: Read a section: Generator function get_expr() yields one section after the other, as a string. An unindented, non-empty line that isn't a comment starts a new section. Lexing: Split section into a list of tokens (strings), with help of generator function tokenize(). Parsing: Parse the first expression from the list of tokens, with parse(), throw away any remaining tokens. In parse_schema(): record value of an enum, union or struct key (if any) in the appropriate global table, append expression to the list of expressions. Return list of expressions. Known issues: (1) Indentation is significant, unlike in real JSON. (2) Neither lexer nor parser have any idea of source positions. Error reporting is hard, let's go shopping. (3) The one error we bother to detect, we "report" via raise. (4) The lexer silently ignores invalid characters. (5) If everything in a section gets ignored, the parser crashes. (6) The lexer treats a string containing a structural character exactly like the structural character. (7) Tokens trailing the first expression in a section are silently ignored. (8) The parser accepts any token in place of a colon. (9) The parser treats comma as optional. (10) parse() crashes on unexpected EOF. (11) parse_schema() crashes when a section's expression isn't a JSON object. Replace this piece of original art by a thoroughly unoriginal design. Takes care of (1), (2), (5), (6) and (7), and lays the groundwork for addressing the others. Generated source files remain unchanged. Signed-off-by: Markus Armbruster <armbru@redhat.com> Reviewed-by: Eric Blake <eblake@redhat.com> Message-id: 1374939721-7876-4-git-send-email-armbru@redhat.com Signed-off-by: Anthony Liguori <aliguori@us.ibm.com>
2013-07-27 17:41:55 +02:00
def get_expr(self, nested):
if self.tok != '{' and not nested:
raise QAPIParseError(self, 'Expected "{"')
qapi.py: Restructure lexer and parser The parser has a rather unorthodox structure: Until EOF: Read a section: Generator function get_expr() yields one section after the other, as a string. An unindented, non-empty line that isn't a comment starts a new section. Lexing: Split section into a list of tokens (strings), with help of generator function tokenize(). Parsing: Parse the first expression from the list of tokens, with parse(), throw away any remaining tokens. In parse_schema(): record value of an enum, union or struct key (if any) in the appropriate global table, append expression to the list of expressions. Return list of expressions. Known issues: (1) Indentation is significant, unlike in real JSON. (2) Neither lexer nor parser have any idea of source positions. Error reporting is hard, let's go shopping. (3) The one error we bother to detect, we "report" via raise. (4) The lexer silently ignores invalid characters. (5) If everything in a section gets ignored, the parser crashes. (6) The lexer treats a string containing a structural character exactly like the structural character. (7) Tokens trailing the first expression in a section are silently ignored. (8) The parser accepts any token in place of a colon. (9) The parser treats comma as optional. (10) parse() crashes on unexpected EOF. (11) parse_schema() crashes when a section's expression isn't a JSON object. Replace this piece of original art by a thoroughly unoriginal design. Takes care of (1), (2), (5), (6) and (7), and lays the groundwork for addressing the others. Generated source files remain unchanged. Signed-off-by: Markus Armbruster <armbru@redhat.com> Reviewed-by: Eric Blake <eblake@redhat.com> Message-id: 1374939721-7876-4-git-send-email-armbru@redhat.com Signed-off-by: Anthony Liguori <aliguori@us.ibm.com>
2013-07-27 17:41:55 +02:00
if self.tok == '{':
self.accept()
expr = self.get_members()
elif self.tok == '[':
self.accept()
expr = self.get_values()
elif self.tok in "'tfn":
qapi.py: Restructure lexer and parser The parser has a rather unorthodox structure: Until EOF: Read a section: Generator function get_expr() yields one section after the other, as a string. An unindented, non-empty line that isn't a comment starts a new section. Lexing: Split section into a list of tokens (strings), with help of generator function tokenize(). Parsing: Parse the first expression from the list of tokens, with parse(), throw away any remaining tokens. In parse_schema(): record value of an enum, union or struct key (if any) in the appropriate global table, append expression to the list of expressions. Return list of expressions. Known issues: (1) Indentation is significant, unlike in real JSON. (2) Neither lexer nor parser have any idea of source positions. Error reporting is hard, let's go shopping. (3) The one error we bother to detect, we "report" via raise. (4) The lexer silently ignores invalid characters. (5) If everything in a section gets ignored, the parser crashes. (6) The lexer treats a string containing a structural character exactly like the structural character. (7) Tokens trailing the first expression in a section are silently ignored. (8) The parser accepts any token in place of a colon. (9) The parser treats comma as optional. (10) parse() crashes on unexpected EOF. (11) parse_schema() crashes when a section's expression isn't a JSON object. Replace this piece of original art by a thoroughly unoriginal design. Takes care of (1), (2), (5), (6) and (7), and lays the groundwork for addressing the others. Generated source files remain unchanged. Signed-off-by: Markus Armbruster <armbru@redhat.com> Reviewed-by: Eric Blake <eblake@redhat.com> Message-id: 1374939721-7876-4-git-send-email-armbru@redhat.com Signed-off-by: Anthony Liguori <aliguori@us.ibm.com>
2013-07-27 17:41:55 +02:00
expr = self.val
self.accept()
else:
raise QAPIParseError(self, 'Expected "{", "[" or string')
qapi.py: Restructure lexer and parser The parser has a rather unorthodox structure: Until EOF: Read a section: Generator function get_expr() yields one section after the other, as a string. An unindented, non-empty line that isn't a comment starts a new section. Lexing: Split section into a list of tokens (strings), with help of generator function tokenize(). Parsing: Parse the first expression from the list of tokens, with parse(), throw away any remaining tokens. In parse_schema(): record value of an enum, union or struct key (if any) in the appropriate global table, append expression to the list of expressions. Return list of expressions. Known issues: (1) Indentation is significant, unlike in real JSON. (2) Neither lexer nor parser have any idea of source positions. Error reporting is hard, let's go shopping. (3) The one error we bother to detect, we "report" via raise. (4) The lexer silently ignores invalid characters. (5) If everything in a section gets ignored, the parser crashes. (6) The lexer treats a string containing a structural character exactly like the structural character. (7) Tokens trailing the first expression in a section are silently ignored. (8) The parser accepts any token in place of a colon. (9) The parser treats comma as optional. (10) parse() crashes on unexpected EOF. (11) parse_schema() crashes when a section's expression isn't a JSON object. Replace this piece of original art by a thoroughly unoriginal design. Takes care of (1), (2), (5), (6) and (7), and lays the groundwork for addressing the others. Generated source files remain unchanged. Signed-off-by: Markus Armbruster <armbru@redhat.com> Reviewed-by: Eric Blake <eblake@redhat.com> Message-id: 1374939721-7876-4-git-send-email-armbru@redhat.com Signed-off-by: Anthony Liguori <aliguori@us.ibm.com>
2013-07-27 17:41:55 +02:00
return expr
qapi: add qapi2texi script As the name suggests, the qapi2texi script converts JSON QAPI description into a texi file suitable for different target formats (info/man/txt/pdf/html...). It parses the following kind of blocks: Free-form: ## # = Section # == Subsection # # Some text foo with *emphasis* # 1. with a list # 2. like that # # And some code: # | $ echo foo # | -> do this # | <- get that # ## Symbol description: ## # @symbol: # # Symbol body ditto ergo sum. Foo bar # baz ding. # # @param1: the frob to frobnicate # @param2: #optional how hard to frobnicate # # Returns: the frobnicated frob. # If frob isn't frobnicatable, GenericError. # # Since: version # Notes: notes, comments can have # - itemized list # - like this # # Example: # # -> { "execute": "quit" } # <- { "return": {} } # ## That's roughly following the following EBNF grammar: api_comment = "##\n" comment "##\n" comment = freeform_comment | symbol_comment freeform_comment = { "# " text "\n" | "#\n" } symbol_comment = "# @" name ":\n" { member | tag_section | freeform_comment } member = "# @" name ':' [ text ] "\n" freeform_comment tag_section = "# " ( "Returns:", "Since:", "Note:", "Notes:", "Example:", "Examples:" ) [ text ] "\n" freeform_comment text = free text with markup Note that the grammar is ambiguous: a line "# @foo:\n" can be parsed both as freeform_comment and as symbol_comment. The actual parser recognizes symbol_comment. See docs/qapi-code-gen.txt for more details. Deficiencies and limitations: - the generated QMP documentation includes internal types - union type support is lacking - type information is lacking in generated documentation - doc comment error message positions are imprecise, they point to the beginning of the comment. - a few minor issues, all marked TODO/FIXME in the code Signed-off-by: Marc-André Lureau <marcandre.lureau@redhat.com> Message-Id: <20170113144135.5150-16-marcandre.lureau@redhat.com> Reviewed-by: Markus Armbruster <armbru@redhat.com> [test-qapi.py tweaked to avoid trailing empty lines in .out] Signed-off-by: Markus Armbruster <armbru@redhat.com>
2017-01-13 15:41:29 +01:00
def get_doc(self, info):
if self.val != '##':
raise QAPIParseError(self, "Junk after '##' at start of "
"documentation comment")
doc = QAPIDoc(self, info)
self.accept(False)
while self.tok == '#':
if self.val.startswith('##'):
# End of doc comment
if self.val != '##':
raise QAPIParseError(self, "Junk after '##' at end of "
"documentation comment")
self.accept()
return doc
else:
doc.append(self.val)
self.accept(False)
raise QAPIParseError(self, "Documentation comment must end with '##'")
#
# Semantic analysis of schema expressions
# TODO fold into QAPISchema
# TODO catching name collisions in generated code would be nice
#
def find_base_members(base):
if isinstance(base, dict):
return base
base_struct_define = find_struct(base)
if not base_struct_define:
return None
return base_struct_define['data']
# Return the qtype of an alternate branch, or None on error.
def find_alternate_member_qtype(qapi_type):
if qapi_type in builtin_types:
return builtin_types[qapi_type]
elif find_struct(qapi_type):
return "QTYPE_QDICT"
elif find_enum(qapi_type):
return "QTYPE_QSTRING"
elif find_union(qapi_type):
return "QTYPE_QDICT"
return None
# Return the discriminator enum define if discriminator is specified as an
# enum type, otherwise return None.
def discriminator_find_enum_define(expr):
base = expr.get('base')
discriminator = expr.get('discriminator')
if not (discriminator and base):
return None
base_members = find_base_members(base)
if not base_members:
return None
discriminator_type = base_members.get(discriminator)
if not discriminator_type:
return None
return find_enum(discriminator_type)
# Names must be letters, numbers, -, and _. They must start with letter,
# except for downstream extensions which must start with __RFQDN_.
# Dots are only valid in the downstream extension prefix.
valid_name = re.compile('^(__[a-zA-Z0-9.-]+_)?'
'[a-zA-Z][a-zA-Z0-9_-]*$')
def check_name(info, source, name, allow_optional=False,
enum_member=False):
global valid_name
membername = name
if not isinstance(name, str):
raise QAPISemError(info, "%s requires a string name" % source)
if name.startswith('*'):
membername = name[1:]
if not allow_optional:
raise QAPISemError(info, "%s does not allow optional name '%s'"
% (source, name))
# Enum members can start with a digit, because the generated C
# code always prefixes it with the enum name
if enum_member and membername[0].isdigit():
membername = 'D' + membername
# Reserve the entire 'q_' namespace for c_name(), and for 'q_empty'
# and 'q_obj_*' implicit type names.
qapi: Reserve 'q_*' and 'has_*' member names c_name() produces names starting with 'q_' when protecting a dictionary member name that would fail to directly compile, but in doing so can cause clashes with any member name already beginning with 'q-' or 'q_'. Likewise, we create a C name 'has_' for any optional member that can clash with any member name beginning with 'has-' or 'has_'. Technically, rather than blindly reserving the namespace, we could try to complain about user names only when an actual collision occurs, or even teach c_name() how to munge names to avoid collisions. But it is not trivial, especially when collisions can occur across multiple types (such as via inheritance or flat unions). Besides, no existing .json files are trying to use these names. So it's easier to just outright forbid the potential for collision. We can always relax things in the future if a real need arises for QMP to express member names that have been forbidden here. 'has_' only has to be reserved for struct/union member names, while 'q_' is reserved everywhere (matching the fact that only members can be optional, while we use c_name() for munging both members and entities). Note that we could relax 'q_' restrictions on entities independently from member names; for example, c_name('qmp_' + 'unix') would result in a different function name than our current 'qmp_' + c_name('unix'). Update and add tests to cover the new error messages. Signed-off-by: Eric Blake <eblake@redhat.com> Message-Id: <1445898903-12082-6-git-send-email-eblake@redhat.com> [Consistently pass protect=False to c_name(); commit message tweaked slightly] Signed-off-by: Markus Armbruster <armbru@redhat.com>
2015-10-26 23:34:44 +01:00
if not valid_name.match(membername) or \
c_name(membername, False).startswith('q_'):
raise QAPISemError(info, "%s uses invalid name '%s'" % (source, name))
def add_name(name, info, meta, implicit=False):
global all_names
check_name(info, "'%s'" % meta, name)
# FIXME should reject names that differ only in '_' vs. '.'
# vs. '-', because they're liable to clash in generated C.
if name in all_names:
raise QAPISemError(info, "%s '%s' is already defined"
% (all_names[name], name))
if not implicit and (name.endswith('Kind') or name.endswith('List')):
raise QAPISemError(info, "%s '%s' should not end in '%s'"
% (meta, name, name[-4:]))
all_names[name] = meta
def add_struct(definition, info):
global struct_types
name = definition['struct']
add_name(name, info, 'struct')
struct_types.append(definition)
def find_struct(name):
global struct_types
for struct in struct_types:
if struct['struct'] == name:
return struct
return None
def add_union(definition, info):
global union_types
name = definition['union']
add_name(name, info, 'union')
union_types.append(definition)
def find_union(name):
global union_types
for union in union_types:
if union['union'] == name:
return union
return None
def add_enum(name, info, enum_values=None, implicit=False):
global enum_types
add_name(name, info, 'enum', implicit)
enum_types.append({"enum_name": name, "enum_values": enum_values})
def find_enum(name):
global enum_types
for enum in enum_types:
if enum['enum_name'] == name:
return enum
return None
def is_enum(name):
return find_enum(name) is not None
def check_type(info, source, value, allow_array=False,
allow_dict=False, allow_optional=False,
allow_metas=[]):
global all_names
if value is None:
return
# Check if array type for value is okay
if isinstance(value, list):
if not allow_array:
raise QAPISemError(info, "%s cannot be an array" % source)
if len(value) != 1 or not isinstance(value[0], str):
raise QAPISemError(info,
"%s: array type must contain single type name" %
source)
value = value[0]
# Check if type name for value is okay
if isinstance(value, str):
if value not in all_names:
raise QAPISemError(info, "%s uses unknown type '%s'"
% (source, value))
if not all_names[value] in allow_metas:
raise QAPISemError(info, "%s cannot use %s type '%s'" %
(source, all_names[value], value))
return
if not allow_dict:
raise QAPISemError(info, "%s should be a type name" % source)
if not isinstance(value, OrderedDict):
raise QAPISemError(info,
"%s should be a dictionary or type name" % source)
# value is a dictionary, check that each member is okay
for (key, arg) in value.items():
check_name(info, "Member of %s" % source, key,
allow_optional=allow_optional)
if c_name(key, False) == 'u' or c_name(key, False).startswith('has_'):
raise QAPISemError(info, "Member of %s uses reserved name '%s'"
% (source, key))
# Todo: allow dictionaries to represent default values of
# an optional argument.
check_type(info, "Member '%s' of %s" % (key, source), arg,
allow_array=True,
allow_metas=['built-in', 'union', 'alternate', 'struct',
'enum'])
def check_command(expr, info):
name = expr['command']
qapi: Implement boxed types for commands/events Turn on the ability to pass command and event arguments in a single boxed parameter, which must name a non-empty type (although the type can be a struct with all optional members). For structs, it makes it possible to pass a single qapi type instead of a breakout of all struct members (useful if the arguments are already in a struct or if the number of members is large); for other complex types, it is now possible to use a union or alternate as the data for a command or event. The empty type may be technically feasible if needed down the road, but it's easier to forbid it now and relax things to allow it later, than it is to allow it now and have to special case how the generated 'q_empty' type is handled (see commit 7ce106a9 for reasons why nothing is generated for the empty type). An alternate type is never considered empty, but now that a boxed type can be either an object or an alternate, we have to provide a trivial QAPISchemaAlternateType.is_empty(). The new call to arg_type.is_empty() during QAPISchemaCommand.check() requires that we first check the type in question; but there is no chance of introducing a cycle since objects do not refer back to commands. We still have a split in syntax checking between ad-hoc parsing up front (merely validates that 'boxed' has a sane value) and during .check() methods (if 'boxed' is set, then 'data' must name a non-empty user-defined type). Generated code is unchanged, as long as no client uses the new feature. Signed-off-by: Eric Blake <eblake@redhat.com> Message-Id: <1468468228-27827-10-git-send-email-eblake@redhat.com> Reviewed-by: Markus Armbruster <armbru@redhat.com> [Test files renamed to *-boxed-*] Signed-off-by: Markus Armbruster <armbru@redhat.com>
2016-07-14 05:50:20 +02:00
boxed = expr.get('boxed', False)
qapi: Implement boxed types for commands/events Turn on the ability to pass command and event arguments in a single boxed parameter, which must name a non-empty type (although the type can be a struct with all optional members). For structs, it makes it possible to pass a single qapi type instead of a breakout of all struct members (useful if the arguments are already in a struct or if the number of members is large); for other complex types, it is now possible to use a union or alternate as the data for a command or event. The empty type may be technically feasible if needed down the road, but it's easier to forbid it now and relax things to allow it later, than it is to allow it now and have to special case how the generated 'q_empty' type is handled (see commit 7ce106a9 for reasons why nothing is generated for the empty type). An alternate type is never considered empty, but now that a boxed type can be either an object or an alternate, we have to provide a trivial QAPISchemaAlternateType.is_empty(). The new call to arg_type.is_empty() during QAPISchemaCommand.check() requires that we first check the type in question; but there is no chance of introducing a cycle since objects do not refer back to commands. We still have a split in syntax checking between ad-hoc parsing up front (merely validates that 'boxed' has a sane value) and during .check() methods (if 'boxed' is set, then 'data' must name a non-empty user-defined type). Generated code is unchanged, as long as no client uses the new feature. Signed-off-by: Eric Blake <eblake@redhat.com> Message-Id: <1468468228-27827-10-git-send-email-eblake@redhat.com> Reviewed-by: Markus Armbruster <armbru@redhat.com> [Test files renamed to *-boxed-*] Signed-off-by: Markus Armbruster <armbru@redhat.com>
2016-07-14 05:50:20 +02:00
args_meta = ['struct']
if boxed:
args_meta += ['union', 'alternate']
check_type(info, "'data' for command '%s'" % name,
qapi: Implement boxed types for commands/events Turn on the ability to pass command and event arguments in a single boxed parameter, which must name a non-empty type (although the type can be a struct with all optional members). For structs, it makes it possible to pass a single qapi type instead of a breakout of all struct members (useful if the arguments are already in a struct or if the number of members is large); for other complex types, it is now possible to use a union or alternate as the data for a command or event. The empty type may be technically feasible if needed down the road, but it's easier to forbid it now and relax things to allow it later, than it is to allow it now and have to special case how the generated 'q_empty' type is handled (see commit 7ce106a9 for reasons why nothing is generated for the empty type). An alternate type is never considered empty, but now that a boxed type can be either an object or an alternate, we have to provide a trivial QAPISchemaAlternateType.is_empty(). The new call to arg_type.is_empty() during QAPISchemaCommand.check() requires that we first check the type in question; but there is no chance of introducing a cycle since objects do not refer back to commands. We still have a split in syntax checking between ad-hoc parsing up front (merely validates that 'boxed' has a sane value) and during .check() methods (if 'boxed' is set, then 'data' must name a non-empty user-defined type). Generated code is unchanged, as long as no client uses the new feature. Signed-off-by: Eric Blake <eblake@redhat.com> Message-Id: <1468468228-27827-10-git-send-email-eblake@redhat.com> Reviewed-by: Markus Armbruster <armbru@redhat.com> [Test files renamed to *-boxed-*] Signed-off-by: Markus Armbruster <armbru@redhat.com>
2016-07-14 05:50:20 +02:00
expr.get('data'), allow_dict=not boxed, allow_optional=True,
allow_metas=args_meta)
returns_meta = ['union', 'struct']
if name in returns_whitelist:
returns_meta += ['built-in', 'alternate', 'enum']
check_type(info, "'returns' for command '%s'" % name,
expr.get('returns'), allow_array=True,
allow_optional=True, allow_metas=returns_meta)
def check_event(expr, info):
global events
name = expr['event']
qapi: Implement boxed types for commands/events Turn on the ability to pass command and event arguments in a single boxed parameter, which must name a non-empty type (although the type can be a struct with all optional members). For structs, it makes it possible to pass a single qapi type instead of a breakout of all struct members (useful if the arguments are already in a struct or if the number of members is large); for other complex types, it is now possible to use a union or alternate as the data for a command or event. The empty type may be technically feasible if needed down the road, but it's easier to forbid it now and relax things to allow it later, than it is to allow it now and have to special case how the generated 'q_empty' type is handled (see commit 7ce106a9 for reasons why nothing is generated for the empty type). An alternate type is never considered empty, but now that a boxed type can be either an object or an alternate, we have to provide a trivial QAPISchemaAlternateType.is_empty(). The new call to arg_type.is_empty() during QAPISchemaCommand.check() requires that we first check the type in question; but there is no chance of introducing a cycle since objects do not refer back to commands. We still have a split in syntax checking between ad-hoc parsing up front (merely validates that 'boxed' has a sane value) and during .check() methods (if 'boxed' is set, then 'data' must name a non-empty user-defined type). Generated code is unchanged, as long as no client uses the new feature. Signed-off-by: Eric Blake <eblake@redhat.com> Message-Id: <1468468228-27827-10-git-send-email-eblake@redhat.com> Reviewed-by: Markus Armbruster <armbru@redhat.com> [Test files renamed to *-boxed-*] Signed-off-by: Markus Armbruster <armbru@redhat.com>
2016-07-14 05:50:20 +02:00
boxed = expr.get('boxed', False)
qapi: Implement boxed types for commands/events Turn on the ability to pass command and event arguments in a single boxed parameter, which must name a non-empty type (although the type can be a struct with all optional members). For structs, it makes it possible to pass a single qapi type instead of a breakout of all struct members (useful if the arguments are already in a struct or if the number of members is large); for other complex types, it is now possible to use a union or alternate as the data for a command or event. The empty type may be technically feasible if needed down the road, but it's easier to forbid it now and relax things to allow it later, than it is to allow it now and have to special case how the generated 'q_empty' type is handled (see commit 7ce106a9 for reasons why nothing is generated for the empty type). An alternate type is never considered empty, but now that a boxed type can be either an object or an alternate, we have to provide a trivial QAPISchemaAlternateType.is_empty(). The new call to arg_type.is_empty() during QAPISchemaCommand.check() requires that we first check the type in question; but there is no chance of introducing a cycle since objects do not refer back to commands. We still have a split in syntax checking between ad-hoc parsing up front (merely validates that 'boxed' has a sane value) and during .check() methods (if 'boxed' is set, then 'data' must name a non-empty user-defined type). Generated code is unchanged, as long as no client uses the new feature. Signed-off-by: Eric Blake <eblake@redhat.com> Message-Id: <1468468228-27827-10-git-send-email-eblake@redhat.com> Reviewed-by: Markus Armbruster <armbru@redhat.com> [Test files renamed to *-boxed-*] Signed-off-by: Markus Armbruster <armbru@redhat.com>
2016-07-14 05:50:20 +02:00
meta = ['struct']
if boxed:
meta += ['union', 'alternate']
events.append(name)
check_type(info, "'data' for event '%s'" % name,
qapi: Implement boxed types for commands/events Turn on the ability to pass command and event arguments in a single boxed parameter, which must name a non-empty type (although the type can be a struct with all optional members). For structs, it makes it possible to pass a single qapi type instead of a breakout of all struct members (useful if the arguments are already in a struct or if the number of members is large); for other complex types, it is now possible to use a union or alternate as the data for a command or event. The empty type may be technically feasible if needed down the road, but it's easier to forbid it now and relax things to allow it later, than it is to allow it now and have to special case how the generated 'q_empty' type is handled (see commit 7ce106a9 for reasons why nothing is generated for the empty type). An alternate type is never considered empty, but now that a boxed type can be either an object or an alternate, we have to provide a trivial QAPISchemaAlternateType.is_empty(). The new call to arg_type.is_empty() during QAPISchemaCommand.check() requires that we first check the type in question; but there is no chance of introducing a cycle since objects do not refer back to commands. We still have a split in syntax checking between ad-hoc parsing up front (merely validates that 'boxed' has a sane value) and during .check() methods (if 'boxed' is set, then 'data' must name a non-empty user-defined type). Generated code is unchanged, as long as no client uses the new feature. Signed-off-by: Eric Blake <eblake@redhat.com> Message-Id: <1468468228-27827-10-git-send-email-eblake@redhat.com> Reviewed-by: Markus Armbruster <armbru@redhat.com> [Test files renamed to *-boxed-*] Signed-off-by: Markus Armbruster <armbru@redhat.com>
2016-07-14 05:50:20 +02:00
expr.get('data'), allow_dict=not boxed, allow_optional=True,
allow_metas=meta)
def check_union(expr, info):
name = expr['union']
base = expr.get('base')
discriminator = expr.get('discriminator')
members = expr['data']
# Two types of unions, determined by discriminator.
# With no discriminator it is a simple union.
if discriminator is None:
enum_define = None
allow_metas = ['built-in', 'union', 'alternate', 'struct', 'enum']
if base is not None:
raise QAPISemError(info, "Simple union '%s' must not have a base" %
name)
# Else, it's a flat union.
else:
# The object must have a string or dictionary 'base'.
check_type(info, "'base' for union '%s'" % name,
base, allow_dict=True, allow_optional=True,
allow_metas=['struct'])
if not base:
raise QAPISemError(info, "Flat union '%s' must have a base"
% name)
base_members = find_base_members(base)
assert base_members
# The value of member 'discriminator' must name a non-optional
# member of the base struct.
check_name(info, "Discriminator of flat union '%s'" % name,
discriminator)
discriminator_type = base_members.get(discriminator)
if not discriminator_type:
raise QAPISemError(info,
"Discriminator '%s' is not a member of base "
"struct '%s'"
% (discriminator, base))
enum_define = find_enum(discriminator_type)
allow_metas = ['struct']
# Do not allow string discriminator
if not enum_define:
raise QAPISemError(info,
"Discriminator '%s' must be of enumeration "
"type" % discriminator)
# Check every branch; don't allow an empty union
if len(members) == 0:
raise QAPISemError(info, "Union '%s' cannot have empty 'data'" % name)
for (key, value) in members.items():
check_name(info, "Member of union '%s'" % name, key)
# Each value must name a known type
check_type(info, "Member '%s' of union '%s'" % (key, name),
value, allow_array=not base, allow_metas=allow_metas)
# If the discriminator names an enum type, then all members
# of 'data' must also be members of the enum type.
if enum_define:
if key not in enum_define['enum_values']:
raise QAPISemError(info,
"Discriminator value '%s' is not found in "
"enum '%s'"
% (key, enum_define["enum_name"]))
# If discriminator is user-defined, ensure all values are covered
if enum_define:
for value in enum_define['enum_values']:
if value not in members.keys():
raise QAPISemError(info, "Union '%s' data missing '%s' branch"
% (name, value))
def check_alternate(expr, info):
name = expr['alternate']
members = expr['data']
types_seen = {}
# Check every branch; require at least two branches
if len(members) < 2:
raise QAPISemError(info,
"Alternate '%s' should have at least two branches "
"in 'data'" % name)
for (key, value) in members.items():
check_name(info, "Member of alternate '%s'" % name, key)
# Ensure alternates have no type conflicts.
check_type(info, "Member '%s' of alternate '%s'" % (key, name),
value,
allow_metas=['built-in', 'union', 'struct', 'enum'])
qtype = find_alternate_member_qtype(value)
if not qtype:
raise QAPISemError(info, "Alternate '%s' member '%s' cannot use "
"type '%s'" % (name, key, value))
if qtype in types_seen:
raise QAPISemError(info, "Alternate '%s' member '%s' can't "
"be distinguished from member '%s'"
% (name, key, types_seen[qtype]))
types_seen[qtype] = key
def check_enum(expr, info):
name = expr['enum']
members = expr.get('data')
prefix = expr.get('prefix')
if not isinstance(members, list):
raise QAPISemError(info,
"Enum '%s' requires an array for 'data'" % name)
if prefix is not None and not isinstance(prefix, str):
raise QAPISemError(info,
"Enum '%s' requires a string for 'prefix'" % name)
for member in members:
check_name(info, "Member of enum '%s'" % name, member,
enum_member=True)
def check_struct(expr, info):
name = expr['struct']
members = expr['data']
check_type(info, "'data' for struct '%s'" % name, members,
allow_dict=True, allow_optional=True)
check_type(info, "'base' for struct '%s'" % name, expr.get('base'),
allow_metas=['struct'])
def check_keys(expr_elem, meta, required, optional=[]):
expr = expr_elem['expr']
info = expr_elem['info']
name = expr[meta]
if not isinstance(name, str):
raise QAPISemError(info, "'%s' key must have a string value" % meta)
required = required + [meta]
for (key, value) in expr.items():
if key not in required and key not in optional:
raise QAPISemError(info, "Unknown key '%s' in %s '%s'"
% (key, meta, name))
if (key == 'gen' or key == 'success-response') and value is not False:
raise QAPISemError(info,
"'%s' of %s '%s' should only use false value"
% (key, meta, name))
qapi: Implement boxed types for commands/events Turn on the ability to pass command and event arguments in a single boxed parameter, which must name a non-empty type (although the type can be a struct with all optional members). For structs, it makes it possible to pass a single qapi type instead of a breakout of all struct members (useful if the arguments are already in a struct or if the number of members is large); for other complex types, it is now possible to use a union or alternate as the data for a command or event. The empty type may be technically feasible if needed down the road, but it's easier to forbid it now and relax things to allow it later, than it is to allow it now and have to special case how the generated 'q_empty' type is handled (see commit 7ce106a9 for reasons why nothing is generated for the empty type). An alternate type is never considered empty, but now that a boxed type can be either an object or an alternate, we have to provide a trivial QAPISchemaAlternateType.is_empty(). The new call to arg_type.is_empty() during QAPISchemaCommand.check() requires that we first check the type in question; but there is no chance of introducing a cycle since objects do not refer back to commands. We still have a split in syntax checking between ad-hoc parsing up front (merely validates that 'boxed' has a sane value) and during .check() methods (if 'boxed' is set, then 'data' must name a non-empty user-defined type). Generated code is unchanged, as long as no client uses the new feature. Signed-off-by: Eric Blake <eblake@redhat.com> Message-Id: <1468468228-27827-10-git-send-email-eblake@redhat.com> Reviewed-by: Markus Armbruster <armbru@redhat.com> [Test files renamed to *-boxed-*] Signed-off-by: Markus Armbruster <armbru@redhat.com>
2016-07-14 05:50:20 +02:00
if key == 'boxed' and value is not True:
raise QAPISemError(info,
"'%s' of %s '%s' should only use true value"
% (key, meta, name))
for key in required:
if key not in expr:
raise QAPISemError(info, "Key '%s' is missing from %s '%s'"
% (key, meta, name))
def check_exprs(exprs):
global all_names
# Learn the types and check for valid expression keys
for builtin in builtin_types.keys():
all_names[builtin] = 'built-in'
for expr_elem in exprs:
expr = expr_elem['expr']
info = expr_elem['info']
qapi: add qapi2texi script As the name suggests, the qapi2texi script converts JSON QAPI description into a texi file suitable for different target formats (info/man/txt/pdf/html...). It parses the following kind of blocks: Free-form: ## # = Section # == Subsection # # Some text foo with *emphasis* # 1. with a list # 2. like that # # And some code: # | $ echo foo # | -> do this # | <- get that # ## Symbol description: ## # @symbol: # # Symbol body ditto ergo sum. Foo bar # baz ding. # # @param1: the frob to frobnicate # @param2: #optional how hard to frobnicate # # Returns: the frobnicated frob. # If frob isn't frobnicatable, GenericError. # # Since: version # Notes: notes, comments can have # - itemized list # - like this # # Example: # # -> { "execute": "quit" } # <- { "return": {} } # ## That's roughly following the following EBNF grammar: api_comment = "##\n" comment "##\n" comment = freeform_comment | symbol_comment freeform_comment = { "# " text "\n" | "#\n" } symbol_comment = "# @" name ":\n" { member | tag_section | freeform_comment } member = "# @" name ':' [ text ] "\n" freeform_comment tag_section = "# " ( "Returns:", "Since:", "Note:", "Notes:", "Example:", "Examples:" ) [ text ] "\n" freeform_comment text = free text with markup Note that the grammar is ambiguous: a line "# @foo:\n" can be parsed both as freeform_comment and as symbol_comment. The actual parser recognizes symbol_comment. See docs/qapi-code-gen.txt for more details. Deficiencies and limitations: - the generated QMP documentation includes internal types - union type support is lacking - type information is lacking in generated documentation - doc comment error message positions are imprecise, they point to the beginning of the comment. - a few minor issues, all marked TODO/FIXME in the code Signed-off-by: Marc-André Lureau <marcandre.lureau@redhat.com> Message-Id: <20170113144135.5150-16-marcandre.lureau@redhat.com> Reviewed-by: Markus Armbruster <armbru@redhat.com> [test-qapi.py tweaked to avoid trailing empty lines in .out] Signed-off-by: Markus Armbruster <armbru@redhat.com>
2017-01-13 15:41:29 +01:00
if 'doc' not in expr_elem:
raise QAPISemError(info,
"Expression missing documentation comment")
if 'enum' in expr:
check_keys(expr_elem, 'enum', ['data'], ['prefix'])
add_enum(expr['enum'], info, expr['data'])
elif 'union' in expr:
check_keys(expr_elem, 'union', ['data'],
['base', 'discriminator'])
add_union(expr, info)
elif 'alternate' in expr:
check_keys(expr_elem, 'alternate', ['data'])
add_name(expr['alternate'], info, 'alternate')
elif 'struct' in expr:
check_keys(expr_elem, 'struct', ['data'], ['base'])
add_struct(expr, info)
elif 'command' in expr:
check_keys(expr_elem, 'command', [],
qapi: Implement boxed types for commands/events Turn on the ability to pass command and event arguments in a single boxed parameter, which must name a non-empty type (although the type can be a struct with all optional members). For structs, it makes it possible to pass a single qapi type instead of a breakout of all struct members (useful if the arguments are already in a struct or if the number of members is large); for other complex types, it is now possible to use a union or alternate as the data for a command or event. The empty type may be technically feasible if needed down the road, but it's easier to forbid it now and relax things to allow it later, than it is to allow it now and have to special case how the generated 'q_empty' type is handled (see commit 7ce106a9 for reasons why nothing is generated for the empty type). An alternate type is never considered empty, but now that a boxed type can be either an object or an alternate, we have to provide a trivial QAPISchemaAlternateType.is_empty(). The new call to arg_type.is_empty() during QAPISchemaCommand.check() requires that we first check the type in question; but there is no chance of introducing a cycle since objects do not refer back to commands. We still have a split in syntax checking between ad-hoc parsing up front (merely validates that 'boxed' has a sane value) and during .check() methods (if 'boxed' is set, then 'data' must name a non-empty user-defined type). Generated code is unchanged, as long as no client uses the new feature. Signed-off-by: Eric Blake <eblake@redhat.com> Message-Id: <1468468228-27827-10-git-send-email-eblake@redhat.com> Reviewed-by: Markus Armbruster <armbru@redhat.com> [Test files renamed to *-boxed-*] Signed-off-by: Markus Armbruster <armbru@redhat.com>
2016-07-14 05:50:20 +02:00
['data', 'returns', 'gen', 'success-response', 'boxed'])
add_name(expr['command'], info, 'command')
elif 'event' in expr:
qapi: Implement boxed types for commands/events Turn on the ability to pass command and event arguments in a single boxed parameter, which must name a non-empty type (although the type can be a struct with all optional members). For structs, it makes it possible to pass a single qapi type instead of a breakout of all struct members (useful if the arguments are already in a struct or if the number of members is large); for other complex types, it is now possible to use a union or alternate as the data for a command or event. The empty type may be technically feasible if needed down the road, but it's easier to forbid it now and relax things to allow it later, than it is to allow it now and have to special case how the generated 'q_empty' type is handled (see commit 7ce106a9 for reasons why nothing is generated for the empty type). An alternate type is never considered empty, but now that a boxed type can be either an object or an alternate, we have to provide a trivial QAPISchemaAlternateType.is_empty(). The new call to arg_type.is_empty() during QAPISchemaCommand.check() requires that we first check the type in question; but there is no chance of introducing a cycle since objects do not refer back to commands. We still have a split in syntax checking between ad-hoc parsing up front (merely validates that 'boxed' has a sane value) and during .check() methods (if 'boxed' is set, then 'data' must name a non-empty user-defined type). Generated code is unchanged, as long as no client uses the new feature. Signed-off-by: Eric Blake <eblake@redhat.com> Message-Id: <1468468228-27827-10-git-send-email-eblake@redhat.com> Reviewed-by: Markus Armbruster <armbru@redhat.com> [Test files renamed to *-boxed-*] Signed-off-by: Markus Armbruster <armbru@redhat.com>
2016-07-14 05:50:20 +02:00
check_keys(expr_elem, 'event', [], ['data', 'boxed'])
add_name(expr['event'], info, 'event')
else:
raise QAPISemError(expr_elem['info'],
"Expression is missing metatype")
# Try again for hidden UnionKind enum
for expr_elem in exprs:
expr = expr_elem['expr']
if 'union' in expr:
if not discriminator_find_enum_define(expr):
add_enum('%sKind' % expr['union'], expr_elem['info'],
implicit=True)
elif 'alternate' in expr:
add_enum('%sKind' % expr['alternate'], expr_elem['info'],
implicit=True)
# Validate that exprs make sense
for expr_elem in exprs:
expr = expr_elem['expr']
info = expr_elem['info']
if 'enum' in expr:
check_enum(expr, info)
elif 'union' in expr:
check_union(expr, info)
elif 'alternate' in expr:
check_alternate(expr, info)
elif 'struct' in expr:
check_struct(expr, info)
elif 'command' in expr:
check_command(expr, info)
elif 'event' in expr:
check_event(expr, info)
else:
assert False, 'unexpected meta type'
return exprs
qapi: add qapi2texi script As the name suggests, the qapi2texi script converts JSON QAPI description into a texi file suitable for different target formats (info/man/txt/pdf/html...). It parses the following kind of blocks: Free-form: ## # = Section # == Subsection # # Some text foo with *emphasis* # 1. with a list # 2. like that # # And some code: # | $ echo foo # | -> do this # | <- get that # ## Symbol description: ## # @symbol: # # Symbol body ditto ergo sum. Foo bar # baz ding. # # @param1: the frob to frobnicate # @param2: #optional how hard to frobnicate # # Returns: the frobnicated frob. # If frob isn't frobnicatable, GenericError. # # Since: version # Notes: notes, comments can have # - itemized list # - like this # # Example: # # -> { "execute": "quit" } # <- { "return": {} } # ## That's roughly following the following EBNF grammar: api_comment = "##\n" comment "##\n" comment = freeform_comment | symbol_comment freeform_comment = { "# " text "\n" | "#\n" } symbol_comment = "# @" name ":\n" { member | tag_section | freeform_comment } member = "# @" name ':' [ text ] "\n" freeform_comment tag_section = "# " ( "Returns:", "Since:", "Note:", "Notes:", "Example:", "Examples:" ) [ text ] "\n" freeform_comment text = free text with markup Note that the grammar is ambiguous: a line "# @foo:\n" can be parsed both as freeform_comment and as symbol_comment. The actual parser recognizes symbol_comment. See docs/qapi-code-gen.txt for more details. Deficiencies and limitations: - the generated QMP documentation includes internal types - union type support is lacking - type information is lacking in generated documentation - doc comment error message positions are imprecise, they point to the beginning of the comment. - a few minor issues, all marked TODO/FIXME in the code Signed-off-by: Marc-André Lureau <marcandre.lureau@redhat.com> Message-Id: <20170113144135.5150-16-marcandre.lureau@redhat.com> Reviewed-by: Markus Armbruster <armbru@redhat.com> [test-qapi.py tweaked to avoid trailing empty lines in .out] Signed-off-by: Markus Armbruster <armbru@redhat.com>
2017-01-13 15:41:29 +01:00
def check_freeform_doc(doc):
if doc.symbol:
raise QAPISemError(doc.info,
"Documention for '%s' is not followed"
" by the definition" % doc.symbol)
body = str(doc.body)
if re.search(r'@\S+:', body, re.MULTILINE):
raise QAPISemError(doc.info,
"Free-form documentation block must not contain"
" @NAME: sections")
def check_definition_doc(doc, expr, info):
for i in ('enum', 'union', 'alternate', 'struct', 'command', 'event'):
if i in expr:
meta = i
break
name = expr[meta]
if doc.symbol != name:
raise QAPISemError(info, "Definition of '%s' follows documentation"
" for '%s'" % (name, doc.symbol))
if doc.has_section('Returns') and 'command' not in expr:
raise QAPISemError(info, "'Returns:' is only valid for commands")
if meta == 'union':
args = expr.get('base', [])
else:
args = expr.get('data', [])
if isinstance(args, str):
return
if isinstance(args, dict):
args = args.keys()
assert isinstance(args, list)
if (meta == 'alternate'
or (meta == 'union' and not expr.get('discriminator'))):
args.append('type')
for arg in args:
if arg[0] == '*':
opt = True
desc = doc.args.get(arg[1:])
else:
opt = False
desc = doc.args.get(arg)
if not desc:
continue
desc_opt = "#optional" in str(desc)
if desc_opt and not opt:
raise QAPISemError(info, "Description has #optional, "
"but the declaration doesn't")
if not desc_opt and opt:
# silently fix the doc
# TODO either fix the schema and make this an error,
# or drop #optional entirely
desc.append("#optional")
doc_args = set(doc.args.keys())
args = set([name.strip('*') for name in args])
if not doc_args.issubset(args):
raise QAPISemError(info, "The following documented members are not in "
"the declaration: %s" % ", ".join(doc_args - args))
def check_docs(docs):
for doc in docs:
for section in doc.args.values() + doc.sections:
content = str(section)
if not content or content.isspace():
raise QAPISemError(doc.info,
"Empty doc section '%s'" % section.name)
if not doc.expr:
check_freeform_doc(doc)
else:
check_definition_doc(doc, doc.expr, doc.info)
return docs
#
# Schema compiler frontend
#
class QAPISchemaEntity(object):
def __init__(self, name, info):
assert isinstance(name, str)
self.name = name
# For explicitly defined entities, info points to the (explicit)
# definition. For builtins (and their arrays), info is None.
# For implicitly defined entities, info points to a place that
# triggered the implicit definition (there may be more than one
# such place).
self.info = info
def c_name(self):
return c_name(self.name)
def check(self, schema):
pass
def is_implicit(self):
return not self.info
def visit(self, visitor):
pass
class QAPISchemaVisitor(object):
def visit_begin(self, schema):
pass
def visit_end(self):
pass
def visit_needed(self, entity):
# Default to visiting everything
return True
def visit_builtin_type(self, name, info, json_type):
pass
def visit_enum_type(self, name, info, values, prefix):
pass
def visit_array_type(self, name, info, element_type):
pass
def visit_object_type(self, name, info, base, members, variants):
pass
qapi: New QMP command query-qmp-schema for QMP introspection qapi/introspect.json defines the introspection schema. It's designed for QMP introspection, but should do for similar uses, such as QGA. The introspection schema does not reflect all the rules and restrictions that apply to QAPI schemata. A valid QAPI schema has an introspection value conforming to the introspection schema, but the converse is not true. Introspection lowers away a number of schema details, and makes implicit things explicit: * The built-in types are declared with their JSON type. All integer types are mapped to 'int', because how many bits we use internally is an implementation detail. It could be pressed into external interface service as very approximate range information, but that's a bad idea. If we need range information, we better do it properly. * Implicit type definitions are made explicit, and given auto-generated names: - Array types, named by appending "List" to the name of their element type, like in generated C. - The enumeration types implicitly defined by simple union types, named by appending "Kind" to the name of their simple union type, like in generated C. - Types that don't occur in generated C. Their names start with ':' so they don't clash with the user's names. * All type references are by name. * The struct and union types are generalized into an object type. * Base types are flattened. * Commands take a single argument and return a single result. Dictionary argument or list result is an implicit type definition. The empty object type is used when a command takes no arguments or produces no results. The argument is always of object type, but the introspection schema doesn't reflect that. The 'gen': false directive is omitted as implementation detail. The 'success-response' directive is omitted as well for now, even though it's not an implementation detail, because it's not used by QMP. * Events carry a single data value. Implicit type definition and empty object type use, just like for commands. The value is of object type, but the introspection schema doesn't reflect that. * Types not used by commands or events are omitted. Indirect use counts as use. * Optional members have a default, which can only be null right now Instead of a mandatory "optional" flag, we have an optional default. No default means mandatory, default null means optional without default value. Non-null is available for optional with default (possible future extension). * Clients should *not* look up types by name, because type names are not ABI. Look up the command or event you're interested in, then follow the references. TODO Should we hide the type names to eliminate the temptation? New generator scripts/qapi-introspect.py computes an introspection value for its input, and generates a C variable holding it. It can generate awfully long lines. Marked TODO. A new test-qmp-input-visitor test case feeds its result for both tests/qapi-schema/qapi-schema-test.json and qapi-schema.json to a QmpInputVisitor to verify it actually conforms to the schema. New QMP command query-qmp-schema takes its return value from that variable. Its reply is some 85KiBytes for me right now. If this turns out to be too much, we have a couple of options: * We can use shorter names in the JSON. Not the QMP style. * Optionally return the sub-schema for commands and events given as arguments. Right now qmp_query_schema() sends the string literal computed by qmp-introspect.py. To compute sub-schema at run time, we'd have to duplicate parts of qapi-introspect.py in C. Unattractive. * Let clients cache the output of query-qmp-schema. It changes only on QEMU upgrades, i.e. rarely. Provide a command query-qmp-schema-hash. Clients can have a cache indexed by hash, and re-query the schema only when they don't have it cached. Even simpler: put the hash in the QMP greeting. Signed-off-by: Markus Armbruster <armbru@redhat.com> Reviewed-by: Eric Blake <eblake@redhat.com>
2015-09-16 13:06:28 +02:00
def visit_object_type_flat(self, name, info, members, variants):
pass
def visit_alternate_type(self, name, info, variants):
pass
def visit_command(self, name, info, arg_type, ret_type,
gen, success_response, boxed):
pass
def visit_event(self, name, info, arg_type, boxed):
pass
class QAPISchemaType(QAPISchemaEntity):
# Return the C type for common use.
# For the types we commonly box, this is a pointer type.
def c_type(self):
pass
# Return the C type to be used in a parameter list.
def c_param_type(self):
return self.c_type()
# Return the C type to be used where we suppress boxing.
def c_unboxed_type(self):
return self.c_type()
def json_type(self):
pass
def alternate_qtype(self):
json2qtype = {
'string': 'QTYPE_QSTRING',
'number': 'QTYPE_QFLOAT',
'int': 'QTYPE_QINT',
'boolean': 'QTYPE_QBOOL',
'object': 'QTYPE_QDICT'
}
return json2qtype.get(self.json_type())
class QAPISchemaBuiltinType(QAPISchemaType):
def __init__(self, name, json_type, c_type):
QAPISchemaType.__init__(self, name, None)
assert not c_type or isinstance(c_type, str)
assert json_type in ('string', 'number', 'int', 'boolean', 'null',
'value')
self._json_type_name = json_type
self._c_type_name = c_type
def c_name(self):
return self.name
def c_type(self):
return self._c_type_name
def c_param_type(self):
if self.name == 'str':
return 'const ' + self._c_type_name
return self._c_type_name
def json_type(self):
return self._json_type_name
def visit(self, visitor):
visitor.visit_builtin_type(self.name, self.info, self.json_type())
class QAPISchemaEnumType(QAPISchemaType):
def __init__(self, name, info, values, prefix):
QAPISchemaType.__init__(self, name, info)
for v in values:
assert isinstance(v, QAPISchemaMember)
v.set_owner(name)
assert prefix is None or isinstance(prefix, str)
self.values = values
self.prefix = prefix
def check(self, schema):
seen = {}
for v in self.values:
v.check_clash(self.info, seen)
def is_implicit(self):
# See QAPISchema._make_implicit_enum_type()
return self.name.endswith('Kind')
def c_type(self):
return c_name(self.name)
def member_names(self):
return [v.name for v in self.values]
def json_type(self):
return 'string'
def visit(self, visitor):
visitor.visit_enum_type(self.name, self.info,
self.member_names(), self.prefix)
class QAPISchemaArrayType(QAPISchemaType):
def __init__(self, name, info, element_type):
QAPISchemaType.__init__(self, name, info)
assert isinstance(element_type, str)
self._element_type_name = element_type
self.element_type = None
def check(self, schema):
self.element_type = schema.lookup_type(self._element_type_name)
assert self.element_type
def is_implicit(self):
return True
def c_type(self):
return c_name(self.name) + pointer_suffix
def json_type(self):
return 'array'
def visit(self, visitor):
visitor.visit_array_type(self.name, self.info, self.element_type)
class QAPISchemaObjectType(QAPISchemaType):
def __init__(self, name, info, base, local_members, variants):
# struct has local_members, optional base, and no variants
# flat union has base, variants, and no local_members
# simple union has local_members, variants, and no base
QAPISchemaType.__init__(self, name, info)
assert base is None or isinstance(base, str)
for m in local_members:
assert isinstance(m, QAPISchemaObjectTypeMember)
qapi: Track owner of each object member Future commits will migrate semantic checking away from parsing and over to the various QAPISchema*.check() methods. But to report an error message about an incorrect semantic use of a member of an object type, it helps to know which type, command, or event owns the member. In particular, when a member is inherited from a base type, it is desirable to associate the member name with the base type (and not the type calling member.check()). Rather than packing additional information into the seen array passed to each member.check() (as in seen[m.name] = {'member':m, 'owner':type}), it is easier to have each member track the name of the owner type in the first place (keeping things simpler with the existing seen[m.name] = m). The new member.owner field is set via a new set_owner() method, called when registering the members and variants arrays with an object or variant type. Track only a name, and not the actual type object, to avoid creating a circular python reference chain. Note that Variants.set_owner() method does not set the owner for the tag_member field; this field is set earlier either as part of an object's non-variant members, or explicitly by alternates. The source information is intended for human consumption in error messages, and a new describe() method is added to access the resulting information. For example, given the qapi: { 'command': 'foo', 'data': { 'string': 'str' } } an implementation of visit_command() that calls arg_type.members[0].describe() will see "'string' (parameter of foo)". To make the human-readable name of implicit types work without duplicating efforts, the describe() method has to reverse the name of implicit types, via the helper _pretty_owner(). No change to generated code. Signed-off-by: Eric Blake <eblake@redhat.com> Message-Id: <1447836791-369-16-git-send-email-eblake@redhat.com> [Incorrect & unused -wrapper case in _pretty_owner() dropped] Signed-off-by: Markus Armbruster <armbru@redhat.com>
2015-11-18 09:52:50 +01:00
m.set_owner(name)
if variants is not None:
assert isinstance(variants, QAPISchemaObjectTypeVariants)
variants.set_owner(name)
self._base_name = base
self.base = None
self.local_members = local_members
self.variants = variants
self.members = None
def check(self, schema):
qapi: Detect base class loops It should be fairly obvious that qapi base classes need to form an acyclic graph, since QMP cannot specify the same key more than once, while base classes are included as flat members alongside other members added by the child. But the old check_member_clash() parser function was not prepared to check for this, and entered an infinite recursion (at least until Python gives up, complaining about nesting too deep). Now that check_member_clash() has been recently removed, attempts at self-inheritance trigger an assertion failure introduced by commit ac88219a. The obvious fix is to turn the assertion into a conditional. This patch includes both the tests (base-cycle-direct and base-cycle-indirect) and the fix, since the .err file output for the unfixed case is not useful (particularly when it was warning about unbounded recursion, as that limit may be platform-specific). We don't need to worry about cycles in flat unions (neither the base type nor the type of a variant can be a union) nor in alternates (alternate branches cannot themselves be an alternate). But if we later allow a union type as a variant, we will still be okay, as QAPISchemaObjectTypeVariants.check() triggers the same QAPISchemaObjectType.check() that will detect any loops. Likewise, we need not worry about the case of diamond inheritance where the same class is used for a flat union base class and one of its variants; either both uses will introduce a collision in trying to insert the same member name twice, or the shared type is empty and changes nothing. Signed-off-by: Eric Blake <eblake@redhat.com> Message-Id: <1449033659-25497-16-git-send-email-eblake@redhat.com> Signed-off-by: Markus Armbruster <armbru@redhat.com>
2015-12-02 06:20:59 +01:00
if self.members is False: # check for cycles
raise QAPISemError(self.info,
"Object %s contains itself" % self.name)
if self.members:
return
self.members = False # mark as being checked
seen = OrderedDict()
if self._base_name:
self.base = schema.lookup_type(self._base_name)
assert isinstance(self.base, QAPISchemaObjectType)
self.base.check(schema)
self.base.check_clash(schema, self.info, seen)
for m in self.local_members:
m.check(schema)
m.check_clash(self.info, seen)
self.members = seen.values()
if self.variants:
self.variants.check(schema, seen)
assert self.variants.tag_member in self.members
self.variants.check_clash(schema, self.info, seen)
# Check that the members of this type do not cause duplicate JSON members,
# and update seen to track the members seen so far. Report any errors
# on behalf of info, which is not necessarily self.info
def check_clash(self, schema, info, seen):
assert not self.variants # not implemented
for m in self.members:
m.check_clash(info, seen)
def is_implicit(self):
# See QAPISchema._make_implicit_object_type(), as well as
# _def_predefineds()
return self.name.startswith('q_')
def is_empty(self):
assert self.members is not None
return not self.members and not self.variants
def c_name(self):
assert self.name != 'q_empty'
return QAPISchemaType.c_name(self)
def c_type(self):
assert not self.is_implicit()
qapi: Don't box struct branch of alternate There's no reason to do two malloc's for an alternate type visiting a QAPI struct; let's just inline the struct directly as the C union branch of the struct. Surprisingly, no clients were actually using the struct member prior to this patch outside of the testsuite; an earlier patch in the series added some testsuite coverage to make the effect of this patch more obvious. In qapi.py, c_type() gains a new is_unboxed flag to control when we are emitting a C struct unboxed within the context of an outer struct (different from our other two modes of usage with no flags for normal local variable declarations, and with is_param for adding 'const' in a parameter list). I don't know if there is any more pythonic way of collapsing the two flags into a single parameter, as we never have a caller setting both flags at once. Ultimately, we want to also unbox branches for QAPI unions, but as that touches a lot more client code, it is better as separate patches. But since unions and alternates share gen_variants(), I had to hack in a way to test if we are visiting an alternate type for setting the is_unboxed flag: look for a non-object branch. This works because alternates have at least two branches, with at most one object branch, while unions have only object branches. The hack will go away in a later patch. The generated code difference to qapi-types.h is relatively small: | struct BlockdevRef { | QType type; | union { /* union tag is @type */ | void *data; |- BlockdevOptions *definition; |+ BlockdevOptions definition; | char *reference; | } u; | }; The corresponding spot in qapi-visit.c calls visit_type_FOO(), which first calls visit_start_struct() to allocate or deallocate the member and handle a layer of {} from the JSON stream, then visits the members. To peel off the indirection and the memory management that comes with it, we inline this call, then suppress allocation / deallocation by passing NULL to visit_start_struct(), and adjust the member visit: | switch ((*obj)->type) { | case QTYPE_QDICT: |- visit_type_BlockdevOptions(v, name, &(*obj)->u.definition, &err); |+ visit_start_struct(v, name, NULL, 0, &err); |+ if (err) { |+ break; |+ } |+ visit_type_BlockdevOptions_fields(v, &(*obj)->u.definition, &err); |+ error_propagate(errp, err); |+ err = NULL; |+ visit_end_struct(v, &err); | break; | case QTYPE_QSTRING: | visit_type_str(v, name, &(*obj)->u.reference, &err); The visit of non-object fields is unchanged. Signed-off-by: Eric Blake <eblake@redhat.com> Message-Id: <1455778109-6278-13-git-send-email-eblake@redhat.com> [Commit message tweaked] Signed-off-by: Markus Armbruster <armbru@redhat.com>
2016-02-18 07:48:26 +01:00
return c_name(self.name) + pointer_suffix
def c_unboxed_type(self):
return c_name(self.name)
def json_type(self):
return 'object'
def visit(self, visitor):
visitor.visit_object_type(self.name, self.info,
self.base, self.local_members, self.variants)
qapi: New QMP command query-qmp-schema for QMP introspection qapi/introspect.json defines the introspection schema. It's designed for QMP introspection, but should do for similar uses, such as QGA. The introspection schema does not reflect all the rules and restrictions that apply to QAPI schemata. A valid QAPI schema has an introspection value conforming to the introspection schema, but the converse is not true. Introspection lowers away a number of schema details, and makes implicit things explicit: * The built-in types are declared with their JSON type. All integer types are mapped to 'int', because how many bits we use internally is an implementation detail. It could be pressed into external interface service as very approximate range information, but that's a bad idea. If we need range information, we better do it properly. * Implicit type definitions are made explicit, and given auto-generated names: - Array types, named by appending "List" to the name of their element type, like in generated C. - The enumeration types implicitly defined by simple union types, named by appending "Kind" to the name of their simple union type, like in generated C. - Types that don't occur in generated C. Their names start with ':' so they don't clash with the user's names. * All type references are by name. * The struct and union types are generalized into an object type. * Base types are flattened. * Commands take a single argument and return a single result. Dictionary argument or list result is an implicit type definition. The empty object type is used when a command takes no arguments or produces no results. The argument is always of object type, but the introspection schema doesn't reflect that. The 'gen': false directive is omitted as implementation detail. The 'success-response' directive is omitted as well for now, even though it's not an implementation detail, because it's not used by QMP. * Events carry a single data value. Implicit type definition and empty object type use, just like for commands. The value is of object type, but the introspection schema doesn't reflect that. * Types not used by commands or events are omitted. Indirect use counts as use. * Optional members have a default, which can only be null right now Instead of a mandatory "optional" flag, we have an optional default. No default means mandatory, default null means optional without default value. Non-null is available for optional with default (possible future extension). * Clients should *not* look up types by name, because type names are not ABI. Look up the command or event you're interested in, then follow the references. TODO Should we hide the type names to eliminate the temptation? New generator scripts/qapi-introspect.py computes an introspection value for its input, and generates a C variable holding it. It can generate awfully long lines. Marked TODO. A new test-qmp-input-visitor test case feeds its result for both tests/qapi-schema/qapi-schema-test.json and qapi-schema.json to a QmpInputVisitor to verify it actually conforms to the schema. New QMP command query-qmp-schema takes its return value from that variable. Its reply is some 85KiBytes for me right now. If this turns out to be too much, we have a couple of options: * We can use shorter names in the JSON. Not the QMP style. * Optionally return the sub-schema for commands and events given as arguments. Right now qmp_query_schema() sends the string literal computed by qmp-introspect.py. To compute sub-schema at run time, we'd have to duplicate parts of qapi-introspect.py in C. Unattractive. * Let clients cache the output of query-qmp-schema. It changes only on QEMU upgrades, i.e. rarely. Provide a command query-qmp-schema-hash. Clients can have a cache indexed by hash, and re-query the schema only when they don't have it cached. Even simpler: put the hash in the QMP greeting. Signed-off-by: Markus Armbruster <armbru@redhat.com> Reviewed-by: Eric Blake <eblake@redhat.com>
2015-09-16 13:06:28 +02:00
visitor.visit_object_type_flat(self.name, self.info,
self.members, self.variants)
class QAPISchemaMember(object):
qapi: Track owner of each object member Future commits will migrate semantic checking away from parsing and over to the various QAPISchema*.check() methods. But to report an error message about an incorrect semantic use of a member of an object type, it helps to know which type, command, or event owns the member. In particular, when a member is inherited from a base type, it is desirable to associate the member name with the base type (and not the type calling member.check()). Rather than packing additional information into the seen array passed to each member.check() (as in seen[m.name] = {'member':m, 'owner':type}), it is easier to have each member track the name of the owner type in the first place (keeping things simpler with the existing seen[m.name] = m). The new member.owner field is set via a new set_owner() method, called when registering the members and variants arrays with an object or variant type. Track only a name, and not the actual type object, to avoid creating a circular python reference chain. Note that Variants.set_owner() method does not set the owner for the tag_member field; this field is set earlier either as part of an object's non-variant members, or explicitly by alternates. The source information is intended for human consumption in error messages, and a new describe() method is added to access the resulting information. For example, given the qapi: { 'command': 'foo', 'data': { 'string': 'str' } } an implementation of visit_command() that calls arg_type.members[0].describe() will see "'string' (parameter of foo)". To make the human-readable name of implicit types work without duplicating efforts, the describe() method has to reverse the name of implicit types, via the helper _pretty_owner(). No change to generated code. Signed-off-by: Eric Blake <eblake@redhat.com> Message-Id: <1447836791-369-16-git-send-email-eblake@redhat.com> [Incorrect & unused -wrapper case in _pretty_owner() dropped] Signed-off-by: Markus Armbruster <armbru@redhat.com>
2015-11-18 09:52:50 +01:00
role = 'member'
def __init__(self, name):
assert isinstance(name, str)
self.name = name
qapi: Track owner of each object member Future commits will migrate semantic checking away from parsing and over to the various QAPISchema*.check() methods. But to report an error message about an incorrect semantic use of a member of an object type, it helps to know which type, command, or event owns the member. In particular, when a member is inherited from a base type, it is desirable to associate the member name with the base type (and not the type calling member.check()). Rather than packing additional information into the seen array passed to each member.check() (as in seen[m.name] = {'member':m, 'owner':type}), it is easier to have each member track the name of the owner type in the first place (keeping things simpler with the existing seen[m.name] = m). The new member.owner field is set via a new set_owner() method, called when registering the members and variants arrays with an object or variant type. Track only a name, and not the actual type object, to avoid creating a circular python reference chain. Note that Variants.set_owner() method does not set the owner for the tag_member field; this field is set earlier either as part of an object's non-variant members, or explicitly by alternates. The source information is intended for human consumption in error messages, and a new describe() method is added to access the resulting information. For example, given the qapi: { 'command': 'foo', 'data': { 'string': 'str' } } an implementation of visit_command() that calls arg_type.members[0].describe() will see "'string' (parameter of foo)". To make the human-readable name of implicit types work without duplicating efforts, the describe() method has to reverse the name of implicit types, via the helper _pretty_owner(). No change to generated code. Signed-off-by: Eric Blake <eblake@redhat.com> Message-Id: <1447836791-369-16-git-send-email-eblake@redhat.com> [Incorrect & unused -wrapper case in _pretty_owner() dropped] Signed-off-by: Markus Armbruster <armbru@redhat.com>
2015-11-18 09:52:50 +01:00
self.owner = None
def set_owner(self, name):
assert not self.owner
self.owner = name
def check_clash(self, info, seen):
cname = c_name(self.name)
if cname.lower() != cname and self.owner not in case_whitelist:
raise QAPISemError(info,
"%s should not use uppercase" % self.describe())
if cname in seen:
raise QAPISemError(info, "%s collides with %s" %
(self.describe(), seen[cname].describe()))
seen[cname] = self
qapi: Track owner of each object member Future commits will migrate semantic checking away from parsing and over to the various QAPISchema*.check() methods. But to report an error message about an incorrect semantic use of a member of an object type, it helps to know which type, command, or event owns the member. In particular, when a member is inherited from a base type, it is desirable to associate the member name with the base type (and not the type calling member.check()). Rather than packing additional information into the seen array passed to each member.check() (as in seen[m.name] = {'member':m, 'owner':type}), it is easier to have each member track the name of the owner type in the first place (keeping things simpler with the existing seen[m.name] = m). The new member.owner field is set via a new set_owner() method, called when registering the members and variants arrays with an object or variant type. Track only a name, and not the actual type object, to avoid creating a circular python reference chain. Note that Variants.set_owner() method does not set the owner for the tag_member field; this field is set earlier either as part of an object's non-variant members, or explicitly by alternates. The source information is intended for human consumption in error messages, and a new describe() method is added to access the resulting information. For example, given the qapi: { 'command': 'foo', 'data': { 'string': 'str' } } an implementation of visit_command() that calls arg_type.members[0].describe() will see "'string' (parameter of foo)". To make the human-readable name of implicit types work without duplicating efforts, the describe() method has to reverse the name of implicit types, via the helper _pretty_owner(). No change to generated code. Signed-off-by: Eric Blake <eblake@redhat.com> Message-Id: <1447836791-369-16-git-send-email-eblake@redhat.com> [Incorrect & unused -wrapper case in _pretty_owner() dropped] Signed-off-by: Markus Armbruster <armbru@redhat.com>
2015-11-18 09:52:50 +01:00
def _pretty_owner(self):
owner = self.owner
if owner.startswith('q_obj_'):
qapi: Track owner of each object member Future commits will migrate semantic checking away from parsing and over to the various QAPISchema*.check() methods. But to report an error message about an incorrect semantic use of a member of an object type, it helps to know which type, command, or event owns the member. In particular, when a member is inherited from a base type, it is desirable to associate the member name with the base type (and not the type calling member.check()). Rather than packing additional information into the seen array passed to each member.check() (as in seen[m.name] = {'member':m, 'owner':type}), it is easier to have each member track the name of the owner type in the first place (keeping things simpler with the existing seen[m.name] = m). The new member.owner field is set via a new set_owner() method, called when registering the members and variants arrays with an object or variant type. Track only a name, and not the actual type object, to avoid creating a circular python reference chain. Note that Variants.set_owner() method does not set the owner for the tag_member field; this field is set earlier either as part of an object's non-variant members, or explicitly by alternates. The source information is intended for human consumption in error messages, and a new describe() method is added to access the resulting information. For example, given the qapi: { 'command': 'foo', 'data': { 'string': 'str' } } an implementation of visit_command() that calls arg_type.members[0].describe() will see "'string' (parameter of foo)". To make the human-readable name of implicit types work without duplicating efforts, the describe() method has to reverse the name of implicit types, via the helper _pretty_owner(). No change to generated code. Signed-off-by: Eric Blake <eblake@redhat.com> Message-Id: <1447836791-369-16-git-send-email-eblake@redhat.com> [Incorrect & unused -wrapper case in _pretty_owner() dropped] Signed-off-by: Markus Armbruster <armbru@redhat.com>
2015-11-18 09:52:50 +01:00
# See QAPISchema._make_implicit_object_type() - reverse the
# mapping there to create a nice human-readable description
owner = owner[6:]
qapi: Track owner of each object member Future commits will migrate semantic checking away from parsing and over to the various QAPISchema*.check() methods. But to report an error message about an incorrect semantic use of a member of an object type, it helps to know which type, command, or event owns the member. In particular, when a member is inherited from a base type, it is desirable to associate the member name with the base type (and not the type calling member.check()). Rather than packing additional information into the seen array passed to each member.check() (as in seen[m.name] = {'member':m, 'owner':type}), it is easier to have each member track the name of the owner type in the first place (keeping things simpler with the existing seen[m.name] = m). The new member.owner field is set via a new set_owner() method, called when registering the members and variants arrays with an object or variant type. Track only a name, and not the actual type object, to avoid creating a circular python reference chain. Note that Variants.set_owner() method does not set the owner for the tag_member field; this field is set earlier either as part of an object's non-variant members, or explicitly by alternates. The source information is intended for human consumption in error messages, and a new describe() method is added to access the resulting information. For example, given the qapi: { 'command': 'foo', 'data': { 'string': 'str' } } an implementation of visit_command() that calls arg_type.members[0].describe() will see "'string' (parameter of foo)". To make the human-readable name of implicit types work without duplicating efforts, the describe() method has to reverse the name of implicit types, via the helper _pretty_owner(). No change to generated code. Signed-off-by: Eric Blake <eblake@redhat.com> Message-Id: <1447836791-369-16-git-send-email-eblake@redhat.com> [Incorrect & unused -wrapper case in _pretty_owner() dropped] Signed-off-by: Markus Armbruster <armbru@redhat.com>
2015-11-18 09:52:50 +01:00
if owner.endswith('-arg'):
return '(parameter of %s)' % owner[:-4]
elif owner.endswith('-base'):
return '(base of %s)' % owner[:-5]
qapi: Track owner of each object member Future commits will migrate semantic checking away from parsing and over to the various QAPISchema*.check() methods. But to report an error message about an incorrect semantic use of a member of an object type, it helps to know which type, command, or event owns the member. In particular, when a member is inherited from a base type, it is desirable to associate the member name with the base type (and not the type calling member.check()). Rather than packing additional information into the seen array passed to each member.check() (as in seen[m.name] = {'member':m, 'owner':type}), it is easier to have each member track the name of the owner type in the first place (keeping things simpler with the existing seen[m.name] = m). The new member.owner field is set via a new set_owner() method, called when registering the members and variants arrays with an object or variant type. Track only a name, and not the actual type object, to avoid creating a circular python reference chain. Note that Variants.set_owner() method does not set the owner for the tag_member field; this field is set earlier either as part of an object's non-variant members, or explicitly by alternates. The source information is intended for human consumption in error messages, and a new describe() method is added to access the resulting information. For example, given the qapi: { 'command': 'foo', 'data': { 'string': 'str' } } an implementation of visit_command() that calls arg_type.members[0].describe() will see "'string' (parameter of foo)". To make the human-readable name of implicit types work without duplicating efforts, the describe() method has to reverse the name of implicit types, via the helper _pretty_owner(). No change to generated code. Signed-off-by: Eric Blake <eblake@redhat.com> Message-Id: <1447836791-369-16-git-send-email-eblake@redhat.com> [Incorrect & unused -wrapper case in _pretty_owner() dropped] Signed-off-by: Markus Armbruster <armbru@redhat.com>
2015-11-18 09:52:50 +01:00
else:
assert owner.endswith('-wrapper')
# Unreachable and not implemented
assert False
if owner.endswith('Kind'):
# See QAPISchema._make_implicit_enum_type()
return '(branch of %s)' % owner[:-4]
qapi: Track owner of each object member Future commits will migrate semantic checking away from parsing and over to the various QAPISchema*.check() methods. But to report an error message about an incorrect semantic use of a member of an object type, it helps to know which type, command, or event owns the member. In particular, when a member is inherited from a base type, it is desirable to associate the member name with the base type (and not the type calling member.check()). Rather than packing additional information into the seen array passed to each member.check() (as in seen[m.name] = {'member':m, 'owner':type}), it is easier to have each member track the name of the owner type in the first place (keeping things simpler with the existing seen[m.name] = m). The new member.owner field is set via a new set_owner() method, called when registering the members and variants arrays with an object or variant type. Track only a name, and not the actual type object, to avoid creating a circular python reference chain. Note that Variants.set_owner() method does not set the owner for the tag_member field; this field is set earlier either as part of an object's non-variant members, or explicitly by alternates. The source information is intended for human consumption in error messages, and a new describe() method is added to access the resulting information. For example, given the qapi: { 'command': 'foo', 'data': { 'string': 'str' } } an implementation of visit_command() that calls arg_type.members[0].describe() will see "'string' (parameter of foo)". To make the human-readable name of implicit types work without duplicating efforts, the describe() method has to reverse the name of implicit types, via the helper _pretty_owner(). No change to generated code. Signed-off-by: Eric Blake <eblake@redhat.com> Message-Id: <1447836791-369-16-git-send-email-eblake@redhat.com> [Incorrect & unused -wrapper case in _pretty_owner() dropped] Signed-off-by: Markus Armbruster <armbru@redhat.com>
2015-11-18 09:52:50 +01:00
return '(%s of %s)' % (self.role, owner)
def describe(self):
return "'%s' %s" % (self.name, self._pretty_owner())
class QAPISchemaObjectTypeMember(QAPISchemaMember):
def __init__(self, name, typ, optional):
QAPISchemaMember.__init__(self, name)
assert isinstance(typ, str)
assert isinstance(optional, bool)
self._type_name = typ
self.type = None
self.optional = optional
def check(self, schema):
assert self.owner
self.type = schema.lookup_type(self._type_name)
assert self.type
class QAPISchemaObjectTypeVariants(object):
def __init__(self, tag_name, tag_member, variants):
# Flat unions pass tag_name but not tag_member.
# Simple unions and alternates pass tag_member but not tag_name.
# After check(), tag_member is always set, and tag_name remains
# a reliable witness of being used by a flat union.
assert bool(tag_member) != bool(tag_name)
assert (isinstance(tag_name, str) or
isinstance(tag_member, QAPISchemaObjectTypeMember))
assert len(variants) > 0
for v in variants:
assert isinstance(v, QAPISchemaObjectTypeVariant)
self._tag_name = tag_name
self.tag_member = tag_member
self.variants = variants
qapi: Track owner of each object member Future commits will migrate semantic checking away from parsing and over to the various QAPISchema*.check() methods. But to report an error message about an incorrect semantic use of a member of an object type, it helps to know which type, command, or event owns the member. In particular, when a member is inherited from a base type, it is desirable to associate the member name with the base type (and not the type calling member.check()). Rather than packing additional information into the seen array passed to each member.check() (as in seen[m.name] = {'member':m, 'owner':type}), it is easier to have each member track the name of the owner type in the first place (keeping things simpler with the existing seen[m.name] = m). The new member.owner field is set via a new set_owner() method, called when registering the members and variants arrays with an object or variant type. Track only a name, and not the actual type object, to avoid creating a circular python reference chain. Note that Variants.set_owner() method does not set the owner for the tag_member field; this field is set earlier either as part of an object's non-variant members, or explicitly by alternates. The source information is intended for human consumption in error messages, and a new describe() method is added to access the resulting information. For example, given the qapi: { 'command': 'foo', 'data': { 'string': 'str' } } an implementation of visit_command() that calls arg_type.members[0].describe() will see "'string' (parameter of foo)". To make the human-readable name of implicit types work without duplicating efforts, the describe() method has to reverse the name of implicit types, via the helper _pretty_owner(). No change to generated code. Signed-off-by: Eric Blake <eblake@redhat.com> Message-Id: <1447836791-369-16-git-send-email-eblake@redhat.com> [Incorrect & unused -wrapper case in _pretty_owner() dropped] Signed-off-by: Markus Armbruster <armbru@redhat.com>
2015-11-18 09:52:50 +01:00
def set_owner(self, name):
for v in self.variants:
v.set_owner(name)
def check(self, schema, seen):
if not self.tag_member: # flat union
self.tag_member = seen[c_name(self._tag_name)]
assert self._tag_name == self.tag_member.name
assert isinstance(self.tag_member.type, QAPISchemaEnumType)
for v in self.variants:
v.check(schema)
qapi: Simplify visiting of alternate types Previously, working with alternates required two lookup arrays and some indirection: for type Foo, we created Foo_qtypes[] which maps each qtype to a value of the generated FooKind enum, then look up that value in FooKind_lookup[] like we do for other union types. This has a couple of subtle bugs. First, the generator was creating a call with a parameter '(int *) &(*obj)->type' where type is an enum type; this is unsafe if the compiler chooses to store the enum type in a different size than int, where assigning through the wrong size pointer can corrupt data or cause a SIGBUS. Related bug, not not fixed in this patch: qapi-visit.py's gen_visit_enum() generates a cast of its enum * argument to int *. Marked FIXME. Second, since the values of the FooKind enum start at zero, all entries of the Foo_qtypes[] array that were not explicitly initialized will map to the same branch of the union as the first member of the alternate, rather than triggering a desired failure in visit_get_next_type(). Fortunately, the bug seldom bites; the very next thing the input visitor does is try to parse the incoming JSON with the wrong parser, which normally fails; the output visitor is not used with a C struct in that state, and the dealloc visitor has nothing to clean up (so there is no leak). However, the second bug IS observable in one case: parsing an integer causes unusual behavior in an alternate that contains at least a 'number' member but no 'int' member, because the 'number' parser accepts QTYPE_QINT in addition to the expected QTYPE_QFLOAT (that is, since 'int' is not a member, the type QTYPE_QINT accidentally maps to FooKind 0; if this enum value is the 'number' branch the integer parses successfully, but if the 'number' branch is not first, some other branch tries to parse the integer and rejects it). A later patch will worry about fixing alternates to always parse all inputs that a non-alternate 'number' would accept, for now this is still marked FIXME in the updated test-qmp-input-visitor.c, to merely point out that new undesired behavior of 'ans' matches the existing undesired behavior of 'asn'. This patch fixes the default-initialization bug by deleting the indirection, and modifying get_next_type() to directly assign a QTypeCode parameter. This in turn fixes the type-casting bug, as we are no longer casting a pointer to enum to a questionable size. There is no longer a need to generate an implicit FooKind enum associated with the alternate type (since the QMP wire format never uses the stringized counterparts of the C union member names). Since the updated visit_get_next_type() does not know which qtypes are expected, the generated visitor is modified to generate an error statement if an unexpected type is encountered. Callers now have to know the QTYPE_* mapping when looking at the discriminator; but so far, only the testsuite was even using the C struct of an alternate types. I considered the possibility of keeping the internal enum FooKind, but initialized differently than most generated arrays, as in: typedef enum FooKind { FOO_KIND_A = QTYPE_QDICT, FOO_KIND_B = QTYPE_QINT, } FooKind; to create nicer aliases for knowing when to use foo->a or foo->b when inspecting foo->type; but it turned out to add too much complexity, especially without a client. There is a user-visible side effect to this change, but I consider it to be an improvement. Previously, the invalid QMP command: {"execute":"blockdev-add", "arguments":{"options": {"driver":"raw", "id":"a", "file":true}}} failed with: {"error": {"class": "GenericError", "desc": "Invalid parameter type for 'file', expected: QDict"}} (visit_get_next_type() succeeded, and the error comes from the visit_type_BlockdevOptions() expecting {}; there is no mention of the fact that a string would also work). Now it fails with: {"error": {"class": "GenericError", "desc": "Invalid parameter type for 'file', expected: BlockdevRef"}} (the error when the next type doesn't match any expected types for the overall alternate). Signed-off-by: Eric Blake <eblake@redhat.com> Message-Id: <1449033659-25497-5-git-send-email-eblake@redhat.com> Signed-off-by: Markus Armbruster <armbru@redhat.com>
2015-12-02 06:20:48 +01:00
# Union names must match enum values; alternate names are
# checked separately. Use 'seen' to tell the two apart.
if seen:
assert v.name in self.tag_member.type.member_names()
qapi: Simplify visiting of alternate types Previously, working with alternates required two lookup arrays and some indirection: for type Foo, we created Foo_qtypes[] which maps each qtype to a value of the generated FooKind enum, then look up that value in FooKind_lookup[] like we do for other union types. This has a couple of subtle bugs. First, the generator was creating a call with a parameter '(int *) &(*obj)->type' where type is an enum type; this is unsafe if the compiler chooses to store the enum type in a different size than int, where assigning through the wrong size pointer can corrupt data or cause a SIGBUS. Related bug, not not fixed in this patch: qapi-visit.py's gen_visit_enum() generates a cast of its enum * argument to int *. Marked FIXME. Second, since the values of the FooKind enum start at zero, all entries of the Foo_qtypes[] array that were not explicitly initialized will map to the same branch of the union as the first member of the alternate, rather than triggering a desired failure in visit_get_next_type(). Fortunately, the bug seldom bites; the very next thing the input visitor does is try to parse the incoming JSON with the wrong parser, which normally fails; the output visitor is not used with a C struct in that state, and the dealloc visitor has nothing to clean up (so there is no leak). However, the second bug IS observable in one case: parsing an integer causes unusual behavior in an alternate that contains at least a 'number' member but no 'int' member, because the 'number' parser accepts QTYPE_QINT in addition to the expected QTYPE_QFLOAT (that is, since 'int' is not a member, the type QTYPE_QINT accidentally maps to FooKind 0; if this enum value is the 'number' branch the integer parses successfully, but if the 'number' branch is not first, some other branch tries to parse the integer and rejects it). A later patch will worry about fixing alternates to always parse all inputs that a non-alternate 'number' would accept, for now this is still marked FIXME in the updated test-qmp-input-visitor.c, to merely point out that new undesired behavior of 'ans' matches the existing undesired behavior of 'asn'. This patch fixes the default-initialization bug by deleting the indirection, and modifying get_next_type() to directly assign a QTypeCode parameter. This in turn fixes the type-casting bug, as we are no longer casting a pointer to enum to a questionable size. There is no longer a need to generate an implicit FooKind enum associated with the alternate type (since the QMP wire format never uses the stringized counterparts of the C union member names). Since the updated visit_get_next_type() does not know which qtypes are expected, the generated visitor is modified to generate an error statement if an unexpected type is encountered. Callers now have to know the QTYPE_* mapping when looking at the discriminator; but so far, only the testsuite was even using the C struct of an alternate types. I considered the possibility of keeping the internal enum FooKind, but initialized differently than most generated arrays, as in: typedef enum FooKind { FOO_KIND_A = QTYPE_QDICT, FOO_KIND_B = QTYPE_QINT, } FooKind; to create nicer aliases for knowing when to use foo->a or foo->b when inspecting foo->type; but it turned out to add too much complexity, especially without a client. There is a user-visible side effect to this change, but I consider it to be an improvement. Previously, the invalid QMP command: {"execute":"blockdev-add", "arguments":{"options": {"driver":"raw", "id":"a", "file":true}}} failed with: {"error": {"class": "GenericError", "desc": "Invalid parameter type for 'file', expected: QDict"}} (visit_get_next_type() succeeded, and the error comes from the visit_type_BlockdevOptions() expecting {}; there is no mention of the fact that a string would also work). Now it fails with: {"error": {"class": "GenericError", "desc": "Invalid parameter type for 'file', expected: BlockdevRef"}} (the error when the next type doesn't match any expected types for the overall alternate). Signed-off-by: Eric Blake <eblake@redhat.com> Message-Id: <1449033659-25497-5-git-send-email-eblake@redhat.com> Signed-off-by: Markus Armbruster <armbru@redhat.com>
2015-12-02 06:20:48 +01:00
assert isinstance(v.type, QAPISchemaObjectType)
qapi: Check for QAPI collisions involving variant members Right now, our ad hoc parser ensures that we cannot have a flat union that introduces any members that would clash with non-variant members inherited from the union's base type (see flat-union-clash-member.json). We want QAPISchemaObjectType.check() to make the same check, so we can later reduce some of the ad hoc checks. We already have a map 'seen' of all non-variant members. We still need to check for collisions between each variant type's members and the non-variant ones. To know the variant type's members, we need to call variant.type.check(). This also detects when a type contains itself in a variant, exactly like the existing base.check() detects when a type contains itself as a base. (Except that we currently forbid anything but a struct as the type of a variant, so we can't actually trigger this type of loop yet.) Slight complication: an alternate's variant can have arbitrary type, but only an object type's check() may be called outside QAPISchema.check(). We could either skip the call for variants of alternates, or skip it for non-object types. For now, do the latter, because it's easier. Then we call each variant member's check_clash() with the appropriate 'seen' map. Since members of different variants can't clash, we have to clone a fresh seen for each variant. Wrap this in a new helper method QAPISchemaObjectTypeVariants.check_clash(). Note that cloning 'seen' inside .check_clash() resembles the one we just removed from .check() in 'qapi: Drop obsolete tag value collision assertions'; the difference here is that we are now checking for clashes among the qapi members of the variant type, rather than for a single clash with the variant tag name itself. Note that, by construction, collisions can't actually happen for simple unions: each variant's type is a wrapper with a single member 'data', which will never collide with the only non-variant member 'type'. For alternates, there's nothing for a variant object type's members to clash with, and therefore no need to call the new variants.check_clash(). No change to generated code. Signed-off-by: Eric Blake <eblake@redhat.com> Message-Id: <1447836791-369-12-git-send-email-eblake@redhat.com> [Commit message tweaked] Signed-off-by: Markus Armbruster <armbru@redhat.com>
2015-11-18 09:52:46 +01:00
v.type.check(schema)
def check_clash(self, schema, info, seen):
qapi: Check for QAPI collisions involving variant members Right now, our ad hoc parser ensures that we cannot have a flat union that introduces any members that would clash with non-variant members inherited from the union's base type (see flat-union-clash-member.json). We want QAPISchemaObjectType.check() to make the same check, so we can later reduce some of the ad hoc checks. We already have a map 'seen' of all non-variant members. We still need to check for collisions between each variant type's members and the non-variant ones. To know the variant type's members, we need to call variant.type.check(). This also detects when a type contains itself in a variant, exactly like the existing base.check() detects when a type contains itself as a base. (Except that we currently forbid anything but a struct as the type of a variant, so we can't actually trigger this type of loop yet.) Slight complication: an alternate's variant can have arbitrary type, but only an object type's check() may be called outside QAPISchema.check(). We could either skip the call for variants of alternates, or skip it for non-object types. For now, do the latter, because it's easier. Then we call each variant member's check_clash() with the appropriate 'seen' map. Since members of different variants can't clash, we have to clone a fresh seen for each variant. Wrap this in a new helper method QAPISchemaObjectTypeVariants.check_clash(). Note that cloning 'seen' inside .check_clash() resembles the one we just removed from .check() in 'qapi: Drop obsolete tag value collision assertions'; the difference here is that we are now checking for clashes among the qapi members of the variant type, rather than for a single clash with the variant tag name itself. Note that, by construction, collisions can't actually happen for simple unions: each variant's type is a wrapper with a single member 'data', which will never collide with the only non-variant member 'type'. For alternates, there's nothing for a variant object type's members to clash with, and therefore no need to call the new variants.check_clash(). No change to generated code. Signed-off-by: Eric Blake <eblake@redhat.com> Message-Id: <1447836791-369-12-git-send-email-eblake@redhat.com> [Commit message tweaked] Signed-off-by: Markus Armbruster <armbru@redhat.com>
2015-11-18 09:52:46 +01:00
for v in self.variants:
# Reset seen map for each variant, since qapi names from one
# branch do not affect another branch
assert isinstance(v.type, QAPISchemaObjectType)
v.type.check_clash(schema, info, dict(seen))
class QAPISchemaObjectTypeVariant(QAPISchemaObjectTypeMember):
qapi: Track owner of each object member Future commits will migrate semantic checking away from parsing and over to the various QAPISchema*.check() methods. But to report an error message about an incorrect semantic use of a member of an object type, it helps to know which type, command, or event owns the member. In particular, when a member is inherited from a base type, it is desirable to associate the member name with the base type (and not the type calling member.check()). Rather than packing additional information into the seen array passed to each member.check() (as in seen[m.name] = {'member':m, 'owner':type}), it is easier to have each member track the name of the owner type in the first place (keeping things simpler with the existing seen[m.name] = m). The new member.owner field is set via a new set_owner() method, called when registering the members and variants arrays with an object or variant type. Track only a name, and not the actual type object, to avoid creating a circular python reference chain. Note that Variants.set_owner() method does not set the owner for the tag_member field; this field is set earlier either as part of an object's non-variant members, or explicitly by alternates. The source information is intended for human consumption in error messages, and a new describe() method is added to access the resulting information. For example, given the qapi: { 'command': 'foo', 'data': { 'string': 'str' } } an implementation of visit_command() that calls arg_type.members[0].describe() will see "'string' (parameter of foo)". To make the human-readable name of implicit types work without duplicating efforts, the describe() method has to reverse the name of implicit types, via the helper _pretty_owner(). No change to generated code. Signed-off-by: Eric Blake <eblake@redhat.com> Message-Id: <1447836791-369-16-git-send-email-eblake@redhat.com> [Incorrect & unused -wrapper case in _pretty_owner() dropped] Signed-off-by: Markus Armbruster <armbru@redhat.com>
2015-11-18 09:52:50 +01:00
role = 'branch'
def __init__(self, name, typ):
QAPISchemaObjectTypeMember.__init__(self, name, typ, False)
class QAPISchemaAlternateType(QAPISchemaType):
def __init__(self, name, info, variants):
QAPISchemaType.__init__(self, name, info)
assert isinstance(variants, QAPISchemaObjectTypeVariants)
assert variants.tag_member
qapi: Track owner of each object member Future commits will migrate semantic checking away from parsing and over to the various QAPISchema*.check() methods. But to report an error message about an incorrect semantic use of a member of an object type, it helps to know which type, command, or event owns the member. In particular, when a member is inherited from a base type, it is desirable to associate the member name with the base type (and not the type calling member.check()). Rather than packing additional information into the seen array passed to each member.check() (as in seen[m.name] = {'member':m, 'owner':type}), it is easier to have each member track the name of the owner type in the first place (keeping things simpler with the existing seen[m.name] = m). The new member.owner field is set via a new set_owner() method, called when registering the members and variants arrays with an object or variant type. Track only a name, and not the actual type object, to avoid creating a circular python reference chain. Note that Variants.set_owner() method does not set the owner for the tag_member field; this field is set earlier either as part of an object's non-variant members, or explicitly by alternates. The source information is intended for human consumption in error messages, and a new describe() method is added to access the resulting information. For example, given the qapi: { 'command': 'foo', 'data': { 'string': 'str' } } an implementation of visit_command() that calls arg_type.members[0].describe() will see "'string' (parameter of foo)". To make the human-readable name of implicit types work without duplicating efforts, the describe() method has to reverse the name of implicit types, via the helper _pretty_owner(). No change to generated code. Signed-off-by: Eric Blake <eblake@redhat.com> Message-Id: <1447836791-369-16-git-send-email-eblake@redhat.com> [Incorrect & unused -wrapper case in _pretty_owner() dropped] Signed-off-by: Markus Armbruster <armbru@redhat.com>
2015-11-18 09:52:50 +01:00
variants.set_owner(name)
variants.tag_member.set_owner(self.name)
self.variants = variants
def check(self, schema):
self.variants.tag_member.check(schema)
qapi: Check for QAPI collisions involving variant members Right now, our ad hoc parser ensures that we cannot have a flat union that introduces any members that would clash with non-variant members inherited from the union's base type (see flat-union-clash-member.json). We want QAPISchemaObjectType.check() to make the same check, so we can later reduce some of the ad hoc checks. We already have a map 'seen' of all non-variant members. We still need to check for collisions between each variant type's members and the non-variant ones. To know the variant type's members, we need to call variant.type.check(). This also detects when a type contains itself in a variant, exactly like the existing base.check() detects when a type contains itself as a base. (Except that we currently forbid anything but a struct as the type of a variant, so we can't actually trigger this type of loop yet.) Slight complication: an alternate's variant can have arbitrary type, but only an object type's check() may be called outside QAPISchema.check(). We could either skip the call for variants of alternates, or skip it for non-object types. For now, do the latter, because it's easier. Then we call each variant member's check_clash() with the appropriate 'seen' map. Since members of different variants can't clash, we have to clone a fresh seen for each variant. Wrap this in a new helper method QAPISchemaObjectTypeVariants.check_clash(). Note that cloning 'seen' inside .check_clash() resembles the one we just removed from .check() in 'qapi: Drop obsolete tag value collision assertions'; the difference here is that we are now checking for clashes among the qapi members of the variant type, rather than for a single clash with the variant tag name itself. Note that, by construction, collisions can't actually happen for simple unions: each variant's type is a wrapper with a single member 'data', which will never collide with the only non-variant member 'type'. For alternates, there's nothing for a variant object type's members to clash with, and therefore no need to call the new variants.check_clash(). No change to generated code. Signed-off-by: Eric Blake <eblake@redhat.com> Message-Id: <1447836791-369-12-git-send-email-eblake@redhat.com> [Commit message tweaked] Signed-off-by: Markus Armbruster <armbru@redhat.com>
2015-11-18 09:52:46 +01:00
# Not calling self.variants.check_clash(), because there's nothing
# to clash with
self.variants.check(schema, {})
qapi: Simplify visiting of alternate types Previously, working with alternates required two lookup arrays and some indirection: for type Foo, we created Foo_qtypes[] which maps each qtype to a value of the generated FooKind enum, then look up that value in FooKind_lookup[] like we do for other union types. This has a couple of subtle bugs. First, the generator was creating a call with a parameter '(int *) &(*obj)->type' where type is an enum type; this is unsafe if the compiler chooses to store the enum type in a different size than int, where assigning through the wrong size pointer can corrupt data or cause a SIGBUS. Related bug, not not fixed in this patch: qapi-visit.py's gen_visit_enum() generates a cast of its enum * argument to int *. Marked FIXME. Second, since the values of the FooKind enum start at zero, all entries of the Foo_qtypes[] array that were not explicitly initialized will map to the same branch of the union as the first member of the alternate, rather than triggering a desired failure in visit_get_next_type(). Fortunately, the bug seldom bites; the very next thing the input visitor does is try to parse the incoming JSON with the wrong parser, which normally fails; the output visitor is not used with a C struct in that state, and the dealloc visitor has nothing to clean up (so there is no leak). However, the second bug IS observable in one case: parsing an integer causes unusual behavior in an alternate that contains at least a 'number' member but no 'int' member, because the 'number' parser accepts QTYPE_QINT in addition to the expected QTYPE_QFLOAT (that is, since 'int' is not a member, the type QTYPE_QINT accidentally maps to FooKind 0; if this enum value is the 'number' branch the integer parses successfully, but if the 'number' branch is not first, some other branch tries to parse the integer and rejects it). A later patch will worry about fixing alternates to always parse all inputs that a non-alternate 'number' would accept, for now this is still marked FIXME in the updated test-qmp-input-visitor.c, to merely point out that new undesired behavior of 'ans' matches the existing undesired behavior of 'asn'. This patch fixes the default-initialization bug by deleting the indirection, and modifying get_next_type() to directly assign a QTypeCode parameter. This in turn fixes the type-casting bug, as we are no longer casting a pointer to enum to a questionable size. There is no longer a need to generate an implicit FooKind enum associated with the alternate type (since the QMP wire format never uses the stringized counterparts of the C union member names). Since the updated visit_get_next_type() does not know which qtypes are expected, the generated visitor is modified to generate an error statement if an unexpected type is encountered. Callers now have to know the QTYPE_* mapping when looking at the discriminator; but so far, only the testsuite was even using the C struct of an alternate types. I considered the possibility of keeping the internal enum FooKind, but initialized differently than most generated arrays, as in: typedef enum FooKind { FOO_KIND_A = QTYPE_QDICT, FOO_KIND_B = QTYPE_QINT, } FooKind; to create nicer aliases for knowing when to use foo->a or foo->b when inspecting foo->type; but it turned out to add too much complexity, especially without a client. There is a user-visible side effect to this change, but I consider it to be an improvement. Previously, the invalid QMP command: {"execute":"blockdev-add", "arguments":{"options": {"driver":"raw", "id":"a", "file":true}}} failed with: {"error": {"class": "GenericError", "desc": "Invalid parameter type for 'file', expected: QDict"}} (visit_get_next_type() succeeded, and the error comes from the visit_type_BlockdevOptions() expecting {}; there is no mention of the fact that a string would also work). Now it fails with: {"error": {"class": "GenericError", "desc": "Invalid parameter type for 'file', expected: BlockdevRef"}} (the error when the next type doesn't match any expected types for the overall alternate). Signed-off-by: Eric Blake <eblake@redhat.com> Message-Id: <1449033659-25497-5-git-send-email-eblake@redhat.com> Signed-off-by: Markus Armbruster <armbru@redhat.com>
2015-12-02 06:20:48 +01:00
# Alternate branch names have no relation to the tag enum values;
# so we have to check for potential name collisions ourselves.
seen = {}
for v in self.variants.variants:
v.check_clash(self.info, seen)
def c_type(self):
return c_name(self.name) + pointer_suffix
def json_type(self):
return 'value'
def visit(self, visitor):
visitor.visit_alternate_type(self.name, self.info, self.variants)
qapi: Implement boxed types for commands/events Turn on the ability to pass command and event arguments in a single boxed parameter, which must name a non-empty type (although the type can be a struct with all optional members). For structs, it makes it possible to pass a single qapi type instead of a breakout of all struct members (useful if the arguments are already in a struct or if the number of members is large); for other complex types, it is now possible to use a union or alternate as the data for a command or event. The empty type may be technically feasible if needed down the road, but it's easier to forbid it now and relax things to allow it later, than it is to allow it now and have to special case how the generated 'q_empty' type is handled (see commit 7ce106a9 for reasons why nothing is generated for the empty type). An alternate type is never considered empty, but now that a boxed type can be either an object or an alternate, we have to provide a trivial QAPISchemaAlternateType.is_empty(). The new call to arg_type.is_empty() during QAPISchemaCommand.check() requires that we first check the type in question; but there is no chance of introducing a cycle since objects do not refer back to commands. We still have a split in syntax checking between ad-hoc parsing up front (merely validates that 'boxed' has a sane value) and during .check() methods (if 'boxed' is set, then 'data' must name a non-empty user-defined type). Generated code is unchanged, as long as no client uses the new feature. Signed-off-by: Eric Blake <eblake@redhat.com> Message-Id: <1468468228-27827-10-git-send-email-eblake@redhat.com> Reviewed-by: Markus Armbruster <armbru@redhat.com> [Test files renamed to *-boxed-*] Signed-off-by: Markus Armbruster <armbru@redhat.com>
2016-07-14 05:50:20 +02:00
def is_empty(self):
return False
class QAPISchemaCommand(QAPISchemaEntity):
def __init__(self, name, info, arg_type, ret_type, gen, success_response,
boxed):
QAPISchemaEntity.__init__(self, name, info)
assert not arg_type or isinstance(arg_type, str)
assert not ret_type or isinstance(ret_type, str)
self._arg_type_name = arg_type
self.arg_type = None
self._ret_type_name = ret_type
self.ret_type = None
self.gen = gen
self.success_response = success_response
self.boxed = boxed
def check(self, schema):
if self._arg_type_name:
self.arg_type = schema.lookup_type(self._arg_type_name)
qapi: Implement boxed types for commands/events Turn on the ability to pass command and event arguments in a single boxed parameter, which must name a non-empty type (although the type can be a struct with all optional members). For structs, it makes it possible to pass a single qapi type instead of a breakout of all struct members (useful if the arguments are already in a struct or if the number of members is large); for other complex types, it is now possible to use a union or alternate as the data for a command or event. The empty type may be technically feasible if needed down the road, but it's easier to forbid it now and relax things to allow it later, than it is to allow it now and have to special case how the generated 'q_empty' type is handled (see commit 7ce106a9 for reasons why nothing is generated for the empty type). An alternate type is never considered empty, but now that a boxed type can be either an object or an alternate, we have to provide a trivial QAPISchemaAlternateType.is_empty(). The new call to arg_type.is_empty() during QAPISchemaCommand.check() requires that we first check the type in question; but there is no chance of introducing a cycle since objects do not refer back to commands. We still have a split in syntax checking between ad-hoc parsing up front (merely validates that 'boxed' has a sane value) and during .check() methods (if 'boxed' is set, then 'data' must name a non-empty user-defined type). Generated code is unchanged, as long as no client uses the new feature. Signed-off-by: Eric Blake <eblake@redhat.com> Message-Id: <1468468228-27827-10-git-send-email-eblake@redhat.com> Reviewed-by: Markus Armbruster <armbru@redhat.com> [Test files renamed to *-boxed-*] Signed-off-by: Markus Armbruster <armbru@redhat.com>
2016-07-14 05:50:20 +02:00
assert (isinstance(self.arg_type, QAPISchemaObjectType) or
isinstance(self.arg_type, QAPISchemaAlternateType))
self.arg_type.check(schema)
if self.boxed:
if self.arg_type.is_empty():
raise QAPISemError(self.info,
"Cannot use 'boxed' with empty type")
qapi: Implement boxed types for commands/events Turn on the ability to pass command and event arguments in a single boxed parameter, which must name a non-empty type (although the type can be a struct with all optional members). For structs, it makes it possible to pass a single qapi type instead of a breakout of all struct members (useful if the arguments are already in a struct or if the number of members is large); for other complex types, it is now possible to use a union or alternate as the data for a command or event. The empty type may be technically feasible if needed down the road, but it's easier to forbid it now and relax things to allow it later, than it is to allow it now and have to special case how the generated 'q_empty' type is handled (see commit 7ce106a9 for reasons why nothing is generated for the empty type). An alternate type is never considered empty, but now that a boxed type can be either an object or an alternate, we have to provide a trivial QAPISchemaAlternateType.is_empty(). The new call to arg_type.is_empty() during QAPISchemaCommand.check() requires that we first check the type in question; but there is no chance of introducing a cycle since objects do not refer back to commands. We still have a split in syntax checking between ad-hoc parsing up front (merely validates that 'boxed' has a sane value) and during .check() methods (if 'boxed' is set, then 'data' must name a non-empty user-defined type). Generated code is unchanged, as long as no client uses the new feature. Signed-off-by: Eric Blake <eblake@redhat.com> Message-Id: <1468468228-27827-10-git-send-email-eblake@redhat.com> Reviewed-by: Markus Armbruster <armbru@redhat.com> [Test files renamed to *-boxed-*] Signed-off-by: Markus Armbruster <armbru@redhat.com>
2016-07-14 05:50:20 +02:00
else:
assert not isinstance(self.arg_type, QAPISchemaAlternateType)
assert not self.arg_type.variants
elif self.boxed:
raise QAPISemError(self.info, "Use of 'boxed' requires 'data'")
if self._ret_type_name:
self.ret_type = schema.lookup_type(self._ret_type_name)
assert isinstance(self.ret_type, QAPISchemaType)
def visit(self, visitor):
visitor.visit_command(self.name, self.info,
self.arg_type, self.ret_type,
self.gen, self.success_response, self.boxed)
class QAPISchemaEvent(QAPISchemaEntity):
def __init__(self, name, info, arg_type, boxed):
QAPISchemaEntity.__init__(self, name, info)
assert not arg_type or isinstance(arg_type, str)
self._arg_type_name = arg_type
self.arg_type = None
self.boxed = boxed
def check(self, schema):
if self._arg_type_name:
self.arg_type = schema.lookup_type(self._arg_type_name)
qapi: Implement boxed types for commands/events Turn on the ability to pass command and event arguments in a single boxed parameter, which must name a non-empty type (although the type can be a struct with all optional members). For structs, it makes it possible to pass a single qapi type instead of a breakout of all struct members (useful if the arguments are already in a struct or if the number of members is large); for other complex types, it is now possible to use a union or alternate as the data for a command or event. The empty type may be technically feasible if needed down the road, but it's easier to forbid it now and relax things to allow it later, than it is to allow it now and have to special case how the generated 'q_empty' type is handled (see commit 7ce106a9 for reasons why nothing is generated for the empty type). An alternate type is never considered empty, but now that a boxed type can be either an object or an alternate, we have to provide a trivial QAPISchemaAlternateType.is_empty(). The new call to arg_type.is_empty() during QAPISchemaCommand.check() requires that we first check the type in question; but there is no chance of introducing a cycle since objects do not refer back to commands. We still have a split in syntax checking between ad-hoc parsing up front (merely validates that 'boxed' has a sane value) and during .check() methods (if 'boxed' is set, then 'data' must name a non-empty user-defined type). Generated code is unchanged, as long as no client uses the new feature. Signed-off-by: Eric Blake <eblake@redhat.com> Message-Id: <1468468228-27827-10-git-send-email-eblake@redhat.com> Reviewed-by: Markus Armbruster <armbru@redhat.com> [Test files renamed to *-boxed-*] Signed-off-by: Markus Armbruster <armbru@redhat.com>
2016-07-14 05:50:20 +02:00
assert (isinstance(self.arg_type, QAPISchemaObjectType) or
isinstance(self.arg_type, QAPISchemaAlternateType))
self.arg_type.check(schema)
if self.boxed:
if self.arg_type.is_empty():
raise QAPISemError(self.info,
"Cannot use 'boxed' with empty type")
qapi: Implement boxed types for commands/events Turn on the ability to pass command and event arguments in a single boxed parameter, which must name a non-empty type (although the type can be a struct with all optional members). For structs, it makes it possible to pass a single qapi type instead of a breakout of all struct members (useful if the arguments are already in a struct or if the number of members is large); for other complex types, it is now possible to use a union or alternate as the data for a command or event. The empty type may be technically feasible if needed down the road, but it's easier to forbid it now and relax things to allow it later, than it is to allow it now and have to special case how the generated 'q_empty' type is handled (see commit 7ce106a9 for reasons why nothing is generated for the empty type). An alternate type is never considered empty, but now that a boxed type can be either an object or an alternate, we have to provide a trivial QAPISchemaAlternateType.is_empty(). The new call to arg_type.is_empty() during QAPISchemaCommand.check() requires that we first check the type in question; but there is no chance of introducing a cycle since objects do not refer back to commands. We still have a split in syntax checking between ad-hoc parsing up front (merely validates that 'boxed' has a sane value) and during .check() methods (if 'boxed' is set, then 'data' must name a non-empty user-defined type). Generated code is unchanged, as long as no client uses the new feature. Signed-off-by: Eric Blake <eblake@redhat.com> Message-Id: <1468468228-27827-10-git-send-email-eblake@redhat.com> Reviewed-by: Markus Armbruster <armbru@redhat.com> [Test files renamed to *-boxed-*] Signed-off-by: Markus Armbruster <armbru@redhat.com>
2016-07-14 05:50:20 +02:00
else:
assert not isinstance(self.arg_type, QAPISchemaAlternateType)
assert not self.arg_type.variants
elif self.boxed:
raise QAPISemError(self.info, "Use of 'boxed' requires 'data'")
def visit(self, visitor):
visitor.visit_event(self.name, self.info, self.arg_type, self.boxed)
class QAPISchema(object):
def __init__(self, fname):
try:
qapi: add qapi2texi script As the name suggests, the qapi2texi script converts JSON QAPI description into a texi file suitable for different target formats (info/man/txt/pdf/html...). It parses the following kind of blocks: Free-form: ## # = Section # == Subsection # # Some text foo with *emphasis* # 1. with a list # 2. like that # # And some code: # | $ echo foo # | -> do this # | <- get that # ## Symbol description: ## # @symbol: # # Symbol body ditto ergo sum. Foo bar # baz ding. # # @param1: the frob to frobnicate # @param2: #optional how hard to frobnicate # # Returns: the frobnicated frob. # If frob isn't frobnicatable, GenericError. # # Since: version # Notes: notes, comments can have # - itemized list # - like this # # Example: # # -> { "execute": "quit" } # <- { "return": {} } # ## That's roughly following the following EBNF grammar: api_comment = "##\n" comment "##\n" comment = freeform_comment | symbol_comment freeform_comment = { "# " text "\n" | "#\n" } symbol_comment = "# @" name ":\n" { member | tag_section | freeform_comment } member = "# @" name ':' [ text ] "\n" freeform_comment tag_section = "# " ( "Returns:", "Since:", "Note:", "Notes:", "Example:", "Examples:" ) [ text ] "\n" freeform_comment text = free text with markup Note that the grammar is ambiguous: a line "# @foo:\n" can be parsed both as freeform_comment and as symbol_comment. The actual parser recognizes symbol_comment. See docs/qapi-code-gen.txt for more details. Deficiencies and limitations: - the generated QMP documentation includes internal types - union type support is lacking - type information is lacking in generated documentation - doc comment error message positions are imprecise, they point to the beginning of the comment. - a few minor issues, all marked TODO/FIXME in the code Signed-off-by: Marc-André Lureau <marcandre.lureau@redhat.com> Message-Id: <20170113144135.5150-16-marcandre.lureau@redhat.com> Reviewed-by: Markus Armbruster <armbru@redhat.com> [test-qapi.py tweaked to avoid trailing empty lines in .out] Signed-off-by: Markus Armbruster <armbru@redhat.com>
2017-01-13 15:41:29 +01:00
parser = QAPISchemaParser(open(fname, "r"))
self.exprs = check_exprs(parser.exprs)
self.docs = check_docs(parser.docs)
self._entity_dict = {}
self._predefining = True
self._def_predefineds()
self._predefining = False
self._def_exprs()
self.check()
except QAPIError as err:
print >>sys.stderr, err
exit(1)
def _def_entity(self, ent):
# Only the predefined types are allowed to not have info
assert ent.info or self._predefining
assert ent.name not in self._entity_dict
self._entity_dict[ent.name] = ent
def lookup_entity(self, name, typ=None):
ent = self._entity_dict.get(name)
if typ and not isinstance(ent, typ):
return None
return ent
def lookup_type(self, name):
return self.lookup_entity(name, QAPISchemaType)
def _def_builtin_type(self, name, json_type, c_type):
self._def_entity(QAPISchemaBuiltinType(name, json_type, c_type))
qapi: Lazy creation of array types Commit ac88219a had several TODO markers about whether we needed to automatically create the corresponding array type alongside any other type. It turns out that most of the time, we don't! There are a few exceptions: 1) We have a few situations where we use an array type in internal code but do not expose that type through QMP; fix it by declaring a dummy type that forces the generator to see that we want to use the array type. 2) The builtin arrays (such as intList for QAPI ['int']) must always be generated, because of the way our QAPI_TYPES_BUILTIN compile guard works: we have situations (at the very least tests/test-qmp-output-visitor.c) that include both top-level "qapi-types.h" (via "error.h") and a secondary "test-qapi-types.h". If we were to only emit the builtin types when used locally, then the first .h file would not include all types, but the second .h does not declare anything at all because the first .h set QAPI_TYPES_BUILTIN, and we would end up with compilation error due to things like unknown type 'int8List'. Actually, we may need to revisit how we do type guards, and change from a single QAPI_TYPES_BUILTIN over to a different usage pattern that does one #ifdef per qapi type - right now, the only types that are declared multiple times between two qapi .json files for inclusion by a single .c file happen to be the builtin arrays. But now that we have QAPI 'include' statements, it is logical to assume that we will soon reach a point where we want to reuse non-builtin types (yes, I'm thinking about what it will take to add introspection to QGA, where we will want to reuse the SchemaInfo type and friends). One #ifdef per type will help ensure that generating the same qapi type into more than one qapi-types.h won't cause collisions when both are included in the same .c file; but we also have to solve how to avoid creating duplicate qapi-types.c entry points. So that is a problem left for another day. Generated code for qapi-types and qapi-visit is drastically reduced; less than a third of the arrays that were blindly created were actually needed (a quick grep shows we dropped from 219 to 69 *List types), and the .o files lost more than 30% of their bulk. [For best results, diff the generated files with 'git diff --patience --no-index pre post'.] Interestingly, the introspection output is unchanged - this is because we already cull all types that are not indirectly reachable from a command or event, so introspection was already using only a subset of array types. The subset of types introspected is now a much larger percentage of the overall set of array types emitted in qapi-types.h (since the larger set shrunk), but still not 100% (evidence that the array types emitted for our new Dummy structs, and the new struct itself, don't affect QMP). Signed-off-by: Eric Blake <eblake@redhat.com> Message-Id: <1444710158-8723-9-git-send-email-eblake@redhat.com> [Moved array info tracking to a later patch] Signed-off-by: Markus Armbruster <armbru@redhat.com>
2015-10-13 06:22:28 +02:00
# TODO As long as we have QAPI_TYPES_BUILTIN to share multiple
# qapi-types.h from a single .c, all arrays of builtins must be
# declared in the first file whether or not they are used. Nicer
# would be to use lazy instantiation, while figuring out how to
# avoid compilation issues with multiple qapi-types.h.
self._make_array_type(name, None)
def _def_predefineds(self):
for t in [('str', 'string', 'char' + pointer_suffix),
('number', 'number', 'double'),
('int', 'int', 'int64_t'),
('int8', 'int', 'int8_t'),
('int16', 'int', 'int16_t'),
('int32', 'int', 'int32_t'),
('int64', 'int', 'int64_t'),
('uint8', 'int', 'uint8_t'),
('uint16', 'int', 'uint16_t'),
('uint32', 'int', 'uint32_t'),
('uint64', 'int', 'uint64_t'),
('size', 'int', 'uint64_t'),
('bool', 'boolean', 'bool'),
('any', 'value', 'QObject' + pointer_suffix)]:
self._def_builtin_type(*t)
self.the_empty_object_type = QAPISchemaObjectType('q_empty', None,
None, [], None)
qapi: New QMP command query-qmp-schema for QMP introspection qapi/introspect.json defines the introspection schema. It's designed for QMP introspection, but should do for similar uses, such as QGA. The introspection schema does not reflect all the rules and restrictions that apply to QAPI schemata. A valid QAPI schema has an introspection value conforming to the introspection schema, but the converse is not true. Introspection lowers away a number of schema details, and makes implicit things explicit: * The built-in types are declared with their JSON type. All integer types are mapped to 'int', because how many bits we use internally is an implementation detail. It could be pressed into external interface service as very approximate range information, but that's a bad idea. If we need range information, we better do it properly. * Implicit type definitions are made explicit, and given auto-generated names: - Array types, named by appending "List" to the name of their element type, like in generated C. - The enumeration types implicitly defined by simple union types, named by appending "Kind" to the name of their simple union type, like in generated C. - Types that don't occur in generated C. Their names start with ':' so they don't clash with the user's names. * All type references are by name. * The struct and union types are generalized into an object type. * Base types are flattened. * Commands take a single argument and return a single result. Dictionary argument or list result is an implicit type definition. The empty object type is used when a command takes no arguments or produces no results. The argument is always of object type, but the introspection schema doesn't reflect that. The 'gen': false directive is omitted as implementation detail. The 'success-response' directive is omitted as well for now, even though it's not an implementation detail, because it's not used by QMP. * Events carry a single data value. Implicit type definition and empty object type use, just like for commands. The value is of object type, but the introspection schema doesn't reflect that. * Types not used by commands or events are omitted. Indirect use counts as use. * Optional members have a default, which can only be null right now Instead of a mandatory "optional" flag, we have an optional default. No default means mandatory, default null means optional without default value. Non-null is available for optional with default (possible future extension). * Clients should *not* look up types by name, because type names are not ABI. Look up the command or event you're interested in, then follow the references. TODO Should we hide the type names to eliminate the temptation? New generator scripts/qapi-introspect.py computes an introspection value for its input, and generates a C variable holding it. It can generate awfully long lines. Marked TODO. A new test-qmp-input-visitor test case feeds its result for both tests/qapi-schema/qapi-schema-test.json and qapi-schema.json to a QmpInputVisitor to verify it actually conforms to the schema. New QMP command query-qmp-schema takes its return value from that variable. Its reply is some 85KiBytes for me right now. If this turns out to be too much, we have a couple of options: * We can use shorter names in the JSON. Not the QMP style. * Optionally return the sub-schema for commands and events given as arguments. Right now qmp_query_schema() sends the string literal computed by qmp-introspect.py. To compute sub-schema at run time, we'd have to duplicate parts of qapi-introspect.py in C. Unattractive. * Let clients cache the output of query-qmp-schema. It changes only on QEMU upgrades, i.e. rarely. Provide a command query-qmp-schema-hash. Clients can have a cache indexed by hash, and re-query the schema only when they don't have it cached. Even simpler: put the hash in the QMP greeting. Signed-off-by: Markus Armbruster <armbru@redhat.com> Reviewed-by: Eric Blake <eblake@redhat.com>
2015-09-16 13:06:28 +02:00
self._def_entity(self.the_empty_object_type)
qtype_values = self._make_enum_members(['none', 'qnull', 'qint',
'qstring', 'qdict', 'qlist',
'qfloat', 'qbool'])
self._def_entity(QAPISchemaEnumType('QType', None, qtype_values,
'QTYPE'))
def _make_enum_members(self, values):
return [QAPISchemaMember(v) for v in values]
def _make_implicit_enum_type(self, name, info, values):
# See also QAPISchemaObjectTypeMember._pretty_owner()
name = name + 'Kind' # Use namespace reserved by add_name()
self._def_entity(QAPISchemaEnumType(
name, info, self._make_enum_members(values), None))
return name
def _make_array_type(self, element_type, info):
name = element_type + 'List' # Use namespace reserved by add_name()
if not self.lookup_type(name):
self._def_entity(QAPISchemaArrayType(name, info, element_type))
return name
def _make_implicit_object_type(self, name, info, role, members):
if not members:
return None
qapi: Track owner of each object member Future commits will migrate semantic checking away from parsing and over to the various QAPISchema*.check() methods. But to report an error message about an incorrect semantic use of a member of an object type, it helps to know which type, command, or event owns the member. In particular, when a member is inherited from a base type, it is desirable to associate the member name with the base type (and not the type calling member.check()). Rather than packing additional information into the seen array passed to each member.check() (as in seen[m.name] = {'member':m, 'owner':type}), it is easier to have each member track the name of the owner type in the first place (keeping things simpler with the existing seen[m.name] = m). The new member.owner field is set via a new set_owner() method, called when registering the members and variants arrays with an object or variant type. Track only a name, and not the actual type object, to avoid creating a circular python reference chain. Note that Variants.set_owner() method does not set the owner for the tag_member field; this field is set earlier either as part of an object's non-variant members, or explicitly by alternates. The source information is intended for human consumption in error messages, and a new describe() method is added to access the resulting information. For example, given the qapi: { 'command': 'foo', 'data': { 'string': 'str' } } an implementation of visit_command() that calls arg_type.members[0].describe() will see "'string' (parameter of foo)". To make the human-readable name of implicit types work without duplicating efforts, the describe() method has to reverse the name of implicit types, via the helper _pretty_owner(). No change to generated code. Signed-off-by: Eric Blake <eblake@redhat.com> Message-Id: <1447836791-369-16-git-send-email-eblake@redhat.com> [Incorrect & unused -wrapper case in _pretty_owner() dropped] Signed-off-by: Markus Armbruster <armbru@redhat.com>
2015-11-18 09:52:50 +01:00
# See also QAPISchemaObjectTypeMember._pretty_owner()
name = 'q_obj_%s-%s' % (name, role)
if not self.lookup_entity(name, QAPISchemaObjectType):
self._def_entity(QAPISchemaObjectType(name, info, None,
members, None))
return name
def _def_enum_type(self, expr, info):
name = expr['enum']
data = expr['data']
prefix = expr.get('prefix')
self._def_entity(QAPISchemaEnumType(
name, info, self._make_enum_members(data), prefix))
def _make_member(self, name, typ, info):
optional = False
if name.startswith('*'):
name = name[1:]
optional = True
if isinstance(typ, list):
assert len(typ) == 1
typ = self._make_array_type(typ[0], info)
return QAPISchemaObjectTypeMember(name, typ, optional)
def _make_members(self, data, info):
return [self._make_member(key, value, info)
for (key, value) in data.iteritems()]
def _def_struct_type(self, expr, info):
name = expr['struct']
base = expr.get('base')
data = expr['data']
self._def_entity(QAPISchemaObjectType(name, info, base,
self._make_members(data, info),
None))
def _make_variant(self, case, typ):
return QAPISchemaObjectTypeVariant(case, typ)
def _make_simple_variant(self, case, typ, info):
if isinstance(typ, list):
assert len(typ) == 1
typ = self._make_array_type(typ[0], info)
typ = self._make_implicit_object_type(
typ, info, 'wrapper', [self._make_member('data', typ, info)])
return QAPISchemaObjectTypeVariant(case, typ)
def _def_union_type(self, expr, info):
name = expr['union']
data = expr['data']
base = expr.get('base')
tag_name = expr.get('discriminator')
tag_member = None
if isinstance(base, dict):
base = (self._make_implicit_object_type(
name, info, 'base', self._make_members(base, info)))
if tag_name:
variants = [self._make_variant(key, value)
for (key, value) in data.iteritems()]
members = []
else:
variants = [self._make_simple_variant(key, value, info)
for (key, value) in data.iteritems()]
typ = self._make_implicit_enum_type(name, info,
[v.name for v in variants])
tag_member = QAPISchemaObjectTypeMember('type', typ, False)
members = [tag_member]
self._def_entity(
QAPISchemaObjectType(name, info, base, members,
QAPISchemaObjectTypeVariants(tag_name,
tag_member,
variants)))
def _def_alternate_type(self, expr, info):
name = expr['alternate']
data = expr['data']
variants = [self._make_variant(key, value)
for (key, value) in data.iteritems()]
qapi: Simplify visiting of alternate types Previously, working with alternates required two lookup arrays and some indirection: for type Foo, we created Foo_qtypes[] which maps each qtype to a value of the generated FooKind enum, then look up that value in FooKind_lookup[] like we do for other union types. This has a couple of subtle bugs. First, the generator was creating a call with a parameter '(int *) &(*obj)->type' where type is an enum type; this is unsafe if the compiler chooses to store the enum type in a different size than int, where assigning through the wrong size pointer can corrupt data or cause a SIGBUS. Related bug, not not fixed in this patch: qapi-visit.py's gen_visit_enum() generates a cast of its enum * argument to int *. Marked FIXME. Second, since the values of the FooKind enum start at zero, all entries of the Foo_qtypes[] array that were not explicitly initialized will map to the same branch of the union as the first member of the alternate, rather than triggering a desired failure in visit_get_next_type(). Fortunately, the bug seldom bites; the very next thing the input visitor does is try to parse the incoming JSON with the wrong parser, which normally fails; the output visitor is not used with a C struct in that state, and the dealloc visitor has nothing to clean up (so there is no leak). However, the second bug IS observable in one case: parsing an integer causes unusual behavior in an alternate that contains at least a 'number' member but no 'int' member, because the 'number' parser accepts QTYPE_QINT in addition to the expected QTYPE_QFLOAT (that is, since 'int' is not a member, the type QTYPE_QINT accidentally maps to FooKind 0; if this enum value is the 'number' branch the integer parses successfully, but if the 'number' branch is not first, some other branch tries to parse the integer and rejects it). A later patch will worry about fixing alternates to always parse all inputs that a non-alternate 'number' would accept, for now this is still marked FIXME in the updated test-qmp-input-visitor.c, to merely point out that new undesired behavior of 'ans' matches the existing undesired behavior of 'asn'. This patch fixes the default-initialization bug by deleting the indirection, and modifying get_next_type() to directly assign a QTypeCode parameter. This in turn fixes the type-casting bug, as we are no longer casting a pointer to enum to a questionable size. There is no longer a need to generate an implicit FooKind enum associated with the alternate type (since the QMP wire format never uses the stringized counterparts of the C union member names). Since the updated visit_get_next_type() does not know which qtypes are expected, the generated visitor is modified to generate an error statement if an unexpected type is encountered. Callers now have to know the QTYPE_* mapping when looking at the discriminator; but so far, only the testsuite was even using the C struct of an alternate types. I considered the possibility of keeping the internal enum FooKind, but initialized differently than most generated arrays, as in: typedef enum FooKind { FOO_KIND_A = QTYPE_QDICT, FOO_KIND_B = QTYPE_QINT, } FooKind; to create nicer aliases for knowing when to use foo->a or foo->b when inspecting foo->type; but it turned out to add too much complexity, especially without a client. There is a user-visible side effect to this change, but I consider it to be an improvement. Previously, the invalid QMP command: {"execute":"blockdev-add", "arguments":{"options": {"driver":"raw", "id":"a", "file":true}}} failed with: {"error": {"class": "GenericError", "desc": "Invalid parameter type for 'file', expected: QDict"}} (visit_get_next_type() succeeded, and the error comes from the visit_type_BlockdevOptions() expecting {}; there is no mention of the fact that a string would also work). Now it fails with: {"error": {"class": "GenericError", "desc": "Invalid parameter type for 'file', expected: BlockdevRef"}} (the error when the next type doesn't match any expected types for the overall alternate). Signed-off-by: Eric Blake <eblake@redhat.com> Message-Id: <1449033659-25497-5-git-send-email-eblake@redhat.com> Signed-off-by: Markus Armbruster <armbru@redhat.com>
2015-12-02 06:20:48 +01:00
tag_member = QAPISchemaObjectTypeMember('type', 'QType', False)
self._def_entity(
QAPISchemaAlternateType(name, info,
QAPISchemaObjectTypeVariants(None,
tag_member,
variants)))
def _def_command(self, expr, info):
name = expr['command']
data = expr.get('data')
rets = expr.get('returns')
gen = expr.get('gen', True)
success_response = expr.get('success-response', True)
boxed = expr.get('boxed', False)
if isinstance(data, OrderedDict):
data = self._make_implicit_object_type(
name, info, 'arg', self._make_members(data, info))
if isinstance(rets, list):
assert len(rets) == 1
rets = self._make_array_type(rets[0], info)
self._def_entity(QAPISchemaCommand(name, info, data, rets, gen,
success_response, boxed))
def _def_event(self, expr, info):
name = expr['event']
data = expr.get('data')
boxed = expr.get('boxed', False)
if isinstance(data, OrderedDict):
data = self._make_implicit_object_type(
name, info, 'arg', self._make_members(data, info))
self._def_entity(QAPISchemaEvent(name, info, data, boxed))
def _def_exprs(self):
for expr_elem in self.exprs:
expr = expr_elem['expr']
info = expr_elem['info']
if 'enum' in expr:
self._def_enum_type(expr, info)
elif 'struct' in expr:
self._def_struct_type(expr, info)
elif 'union' in expr:
self._def_union_type(expr, info)
elif 'alternate' in expr:
self._def_alternate_type(expr, info)
elif 'command' in expr:
self._def_command(expr, info)
elif 'event' in expr:
self._def_event(expr, info)
else:
assert False
def check(self):
for ent in self._entity_dict.values():
ent.check(self)
def visit(self, visitor):
visitor.visit_begin(self)
for (name, entity) in sorted(self._entity_dict.items()):
if visitor.visit_needed(entity):
entity.visit(visitor)
visitor.visit_end()
#
# Code generation helpers
#
def camel_case(name):
new_name = ''
first = True
for ch in name:
if ch in ['_', '-']:
first = True
elif first:
new_name += ch.upper()
first = False
else:
new_name += ch.lower()
return new_name
# ENUMName -> ENUM_NAME, EnumName1 -> ENUM_NAME1
# ENUM_NAME -> ENUM_NAME, ENUM_NAME1 -> ENUM_NAME1, ENUM_Name2 -> ENUM_NAME2
# ENUM24_Name -> ENUM24_NAME
def camel_to_upper(value):
c_fun_str = c_name(value, False)
if value.isupper():
return c_fun_str
new_name = ''
l = len(c_fun_str)
for i in range(l):
c = c_fun_str[i]
# When c is upper and no "_" appears before, do more checks
if c.isupper() and (i > 0) and c_fun_str[i - 1] != "_":
if i < l - 1 and c_fun_str[i + 1].islower():
new_name += '_'
elif c_fun_str[i - 1].isdigit():
new_name += '_'
new_name += c
return new_name.lstrip('_').upper()
def c_enum_const(type_name, const_name, prefix=None):
if prefix is not None:
type_name = prefix
qapi: Change munging of CamelCase enum values When munging enum values, the fact that we were passing the entire prefix + value through camel_to_upper() meant that enum values spelled with CamelCase could be turned into CAMEL_CASE. However, this provides a potential collision (both OneTwo and One-Two would munge into ONE_TWO) for enum types, when the same two names are valid side-by-side as QAPI member names. By changing the generation of enum constants to always be prefix + '_' + c_name(value, False).upper(), and ensuring that there are no case collisions (in the next patches), we no longer have to worry about names that would be distinct as QAPI members but collide as variant tag names, without having to think about what munging the heuristics in camel_to_upper() will actually perform on an enum value. Making the change will affect enums that did not follow coding conventions, using 'CamelCase' rather than desired 'lower-case'. Thankfully, there are only two culprits: InputButton and ErrorClass. We already tweaked ErrorClass to make it an alias of QapiErrorClass, where only the alias needs changing rather than the whole tree. So the bulk of this change is modifying INPUT_BUTTON_WHEEL_UP to the new INPUT_BUTTON_WHEELUP (and likewise for WHEELDOWN). That part of this commit may later need reverting if we rename the enum constants from 'WheelUp' to 'wheel-up' as part of moving x-input-send-event to a stable interface; but at least we have documentation bread crumbs in place to remind us (commit 513e7cd), and it matches the fact that SDL constants are also spelled SDL_BUTTON_WHEELUP. Suggested by: Markus Armbruster <armbru@redhat.com> Signed-off-by: Eric Blake <eblake@redhat.com> Message-Id: <1447836791-369-27-git-send-email-eblake@redhat.com> [Commit message tweaked] Signed-off-by: Markus Armbruster <armbru@redhat.com>
2015-11-18 09:53:01 +01:00
return camel_to_upper(type_name) + '_' + c_name(const_name, False).upper()
c_name_trans = string.maketrans('.-', '__')
# Map @name to a valid C identifier.
# If @protect, avoid returning certain ticklish identifiers (like
# C keywords) by prepending "q_".
#
# Used for converting 'name' from a 'name':'type' qapi definition
# into a generated struct member, as well as converting type names
# into substrings of a generated C function name.
# '__a.b_c' -> '__a_b_c', 'x-foo' -> 'x_foo'
# protect=True: 'int' -> 'q_int'; protect=False: 'int' -> 'int'
def c_name(name, protect=True):
# ANSI X3J11/88-090, 3.1.1
c89_words = set(['auto', 'break', 'case', 'char', 'const', 'continue',
'default', 'do', 'double', 'else', 'enum', 'extern',
'float', 'for', 'goto', 'if', 'int', 'long', 'register',
'return', 'short', 'signed', 'sizeof', 'static',
'struct', 'switch', 'typedef', 'union', 'unsigned',
'void', 'volatile', 'while'])
# ISO/IEC 9899:1999, 6.4.1
c99_words = set(['inline', 'restrict', '_Bool', '_Complex', '_Imaginary'])
# ISO/IEC 9899:2011, 6.4.1
c11_words = set(['_Alignas', '_Alignof', '_Atomic', '_Generic',
'_Noreturn', '_Static_assert', '_Thread_local'])
# GCC http://gcc.gnu.org/onlinedocs/gcc-4.7.1/gcc/C-Extensions.html
# excluding _.*
gcc_words = set(['asm', 'typeof'])
# C++ ISO/IEC 14882:2003 2.11
cpp_words = set(['bool', 'catch', 'class', 'const_cast', 'delete',
'dynamic_cast', 'explicit', 'false', 'friend', 'mutable',
'namespace', 'new', 'operator', 'private', 'protected',
'public', 'reinterpret_cast', 'static_cast', 'template',
'this', 'throw', 'true', 'try', 'typeid', 'typename',
'using', 'virtual', 'wchar_t',
# alternative representations
'and', 'and_eq', 'bitand', 'bitor', 'compl', 'not',
'not_eq', 'or', 'or_eq', 'xor', 'xor_eq'])
# namespace pollution:
polluted_words = set(['unix', 'errno', 'mips', 'sparc'])
name = name.translate(c_name_trans)
if protect and (name in c89_words | c99_words | c11_words | gcc_words
| cpp_words | polluted_words):
return "q_" + name
return name
eatspace = '\033EATSPACE.'
pointer_suffix = ' *' + eatspace
def genindent(count):
ret = ""
for _ in range(count):
ret += " "
return ret
indent_level = 0
def push_indent(indent_amount=4):
global indent_level
indent_level += indent_amount
def pop_indent(indent_amount=4):
global indent_level
indent_level -= indent_amount
# Generate @code with @kwds interpolated.
# Obey indent_level, and strip eatspace.
def cgen(code, **kwds):
raw = code % kwds
if indent_level:
indent = genindent(indent_level)
# re.subn() lacks flags support before Python 2.7, use re.compile()
raw = re.subn(re.compile("^.", re.MULTILINE),
indent + r'\g<0>', raw)
raw = raw[0]
return re.sub(re.escape(eatspace) + ' *', '', raw)
def mcgen(code, **kwds):
if code[0] == '\n':
code = code[1:]
return cgen(code, **kwds)
def guardname(filename):
return c_name(filename, protect=False).upper()
def guardstart(name):
return mcgen('''
#ifndef %(name)s
#define %(name)s
''',
name=guardname(name))
def guardend(name):
return mcgen('''
#endif /* %(name)s */
''',
name=guardname(name))
def gen_enum_lookup(name, values, prefix=None):
ret = mcgen('''
const char *const %(c_name)s_lookup[] = {
''',
c_name=c_name(name))
for value in values:
index = c_enum_const(name, value, prefix)
ret += mcgen('''
[%(index)s] = "%(value)s",
''',
index=index, value=value)
max_index = c_enum_const(name, '_MAX', prefix)
ret += mcgen('''
[%(max_index)s] = NULL,
};
''',
max_index=max_index)
return ret
def gen_enum(name, values, prefix=None):
# append automatically generated _MAX value
enum_values = values + ['_MAX']
ret = mcgen('''
typedef enum %(c_name)s {
''',
c_name=c_name(name))
i = 0
for value in enum_values:
ret += mcgen('''
%(c_enum)s = %(i)d,
''',
c_enum=c_enum_const(name, value, prefix),
i=i)
i += 1
ret += mcgen('''
} %(c_name)s;
''',
c_name=c_name(name))
ret += mcgen('''
extern const char *const %(c_name)s_lookup[];
''',
c_name=c_name(name))
return ret
def gen_params(arg_type, boxed, extra):
if not arg_type:
qapi: Implement boxed types for commands/events Turn on the ability to pass command and event arguments in a single boxed parameter, which must name a non-empty type (although the type can be a struct with all optional members). For structs, it makes it possible to pass a single qapi type instead of a breakout of all struct members (useful if the arguments are already in a struct or if the number of members is large); for other complex types, it is now possible to use a union or alternate as the data for a command or event. The empty type may be technically feasible if needed down the road, but it's easier to forbid it now and relax things to allow it later, than it is to allow it now and have to special case how the generated 'q_empty' type is handled (see commit 7ce106a9 for reasons why nothing is generated for the empty type). An alternate type is never considered empty, but now that a boxed type can be either an object or an alternate, we have to provide a trivial QAPISchemaAlternateType.is_empty(). The new call to arg_type.is_empty() during QAPISchemaCommand.check() requires that we first check the type in question; but there is no chance of introducing a cycle since objects do not refer back to commands. We still have a split in syntax checking between ad-hoc parsing up front (merely validates that 'boxed' has a sane value) and during .check() methods (if 'boxed' is set, then 'data' must name a non-empty user-defined type). Generated code is unchanged, as long as no client uses the new feature. Signed-off-by: Eric Blake <eblake@redhat.com> Message-Id: <1468468228-27827-10-git-send-email-eblake@redhat.com> Reviewed-by: Markus Armbruster <armbru@redhat.com> [Test files renamed to *-boxed-*] Signed-off-by: Markus Armbruster <armbru@redhat.com>
2016-07-14 05:50:20 +02:00
assert not boxed
return extra
ret = ''
sep = ''
if boxed:
qapi: Implement boxed types for commands/events Turn on the ability to pass command and event arguments in a single boxed parameter, which must name a non-empty type (although the type can be a struct with all optional members). For structs, it makes it possible to pass a single qapi type instead of a breakout of all struct members (useful if the arguments are already in a struct or if the number of members is large); for other complex types, it is now possible to use a union or alternate as the data for a command or event. The empty type may be technically feasible if needed down the road, but it's easier to forbid it now and relax things to allow it later, than it is to allow it now and have to special case how the generated 'q_empty' type is handled (see commit 7ce106a9 for reasons why nothing is generated for the empty type). An alternate type is never considered empty, but now that a boxed type can be either an object or an alternate, we have to provide a trivial QAPISchemaAlternateType.is_empty(). The new call to arg_type.is_empty() during QAPISchemaCommand.check() requires that we first check the type in question; but there is no chance of introducing a cycle since objects do not refer back to commands. We still have a split in syntax checking between ad-hoc parsing up front (merely validates that 'boxed' has a sane value) and during .check() methods (if 'boxed' is set, then 'data' must name a non-empty user-defined type). Generated code is unchanged, as long as no client uses the new feature. Signed-off-by: Eric Blake <eblake@redhat.com> Message-Id: <1468468228-27827-10-git-send-email-eblake@redhat.com> Reviewed-by: Markus Armbruster <armbru@redhat.com> [Test files renamed to *-boxed-*] Signed-off-by: Markus Armbruster <armbru@redhat.com>
2016-07-14 05:50:20 +02:00
ret += '%s arg' % arg_type.c_param_type()
sep = ', '
else:
assert not arg_type.variants
for memb in arg_type.members:
ret += sep
sep = ', '
if memb.optional:
ret += 'bool has_%s, ' % c_name(memb.name)
ret += '%s %s' % (memb.type.c_param_type(),
c_name(memb.name))
if extra:
ret += sep + extra
return ret
#
# Common command line parsing
#
def parse_command_line(extra_options="", extra_long_options=[]):
try:
opts, args = getopt.gnu_getopt(sys.argv[1:],
"chp:o:" + extra_options,
["source", "header", "prefix=",
"output-dir="] + extra_long_options)
except getopt.GetoptError as err:
print >>sys.stderr, "%s: %s" % (sys.argv[0], str(err))
sys.exit(1)
output_dir = ""
prefix = ""
do_c = False
do_h = False
extra_opts = []
for oa in opts:
o, a = oa
if o in ("-p", "--prefix"):
match = re.match('([A-Za-z_.-][A-Za-z0-9_.-]*)?', a)
if match.end() != len(a):
print >>sys.stderr, \
"%s: 'funny character '%s' in argument of --prefix" \
% (sys.argv[0], a[match.end()])
sys.exit(1)
prefix = a
elif o in ("-o", "--output-dir"):
output_dir = a + "/"
elif o in ("-c", "--source"):
do_c = True
elif o in ("-h", "--header"):
do_h = True
else:
extra_opts.append(oa)
if not do_c and not do_h:
do_c = True
do_h = True
if len(args) != 1:
print >>sys.stderr, "%s: need exactly one argument" % sys.argv[0]
sys.exit(1)
fname = args[0]
return (fname, output_dir, do_c, do_h, prefix, extra_opts)
#
# Generate output files with boilerplate
#
def open_output(output_dir, do_c, do_h, prefix, c_file, h_file,
c_comment, h_comment):
guard = guardname(prefix + h_file)
c_file = output_dir + prefix + c_file
h_file = output_dir + prefix + h_file
if output_dir:
try:
os.makedirs(output_dir)
except os.error as e:
if e.errno != errno.EEXIST:
raise
def maybe_open(really, name, opt):
if really:
return open(name, opt)
else:
import StringIO
return StringIO.StringIO()
fdef = maybe_open(do_c, c_file, 'w')
fdecl = maybe_open(do_h, h_file, 'w')
fdef.write(mcgen('''
/* AUTOMATICALLY GENERATED, DO NOT MODIFY */
%(comment)s
''',
comment=c_comment))
fdecl.write(mcgen('''
/* AUTOMATICALLY GENERATED, DO NOT MODIFY */
%(comment)s
#ifndef %(guard)s
#define %(guard)s
''',
comment=h_comment, guard=guard))
return (fdef, fdecl)
def close_output(fdef, fdecl):
fdecl.write('''
#endif
''')
fdecl.close()
fdef.close()