OpenE2K/qemu-e2k
13
4
Fork 0
Code Issues 4 Pull Requests Projects 2 Releases Activity

json-parser: don't replicate tokens at each level of recursion

Browse Source 
Currently, when parsing a stream of tokens we make a copy of the token
list at the beginning of each level of recursion so that we do not
modify the original list in cases where we need to fall back to an
earlier state.

In the worst case, we will only read 1 or 2 tokens off the list before
recursing again, which means an upper bound of roughly N^2 token allocations.

For a "reasonably" sized QMP request (in this a QMP representation of
cirrus_vga's device state, generated via QIDL, being passed in via
qom-set), this caused my 16GB's of memory to be exhausted before any
noticeable progress was made by the parser.

This patch works around the issue by using single copy of the token list
in the form of an indexable array so that we can save/restore state by
manipulating indices.

A subsequent commit adds a "large_dict" test case which exhibits the
same behavior as above. With this patch applied the test case successfully
completes in under a second.

Tested with valgrind, make check, and QMP.

Reviewed-by: Eric Blake <eblake@redhat.com>
Signed-off-by: Michael Roth <mdroth@linux.vnet.ibm.com>
Signed-off-by: Anthony Liguori <aliguori@us.ibm.com>
This commit is contained in:
Michael Roth 2012-08-15 13:45:43 -05:00 committed by Anthony Liguori
parent a86a4c2f7b
commit 65c0f1e955
1 changed files with 142 additions and 88 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

230
json-parser.c
View File

@ -27,6 +27,11 @@
typedef struct JSONParserContext typedef struct JSONParserContext
{ {
Error *err; Error *err;
struct {
QObject **buf;
size_t pos;
size_t count;
} tokens;
} JSONParserContext; } JSONParserContext;
#define BUG_ON(cond) assert(!(cond)) #define BUG_ON(cond) assert(!(cond))
@ -40,7 +45,7 @@ typedef struct JSONParserContext
* 4) deal with premature EOI * 4) deal with premature EOI
*/ */
static QObject *parse_value(JSONParserContext *ctxt, QList **tokens, va_list *ap); static QObject *parse_value(JSONParserContext *ctxt, va_list *ap);
/** /**
* Token manipulators * Token manipulators
@ -270,27 +275,111 @@ out:
return NULL; return NULL;
} }
static QObject *parser_context_pop_token(JSONParserContext *ctxt)
{
QObject *token;
g_assert(ctxt->tokens.pos < ctxt->tokens.count);
token = ctxt->tokens.buf[ctxt->tokens.pos];
ctxt->tokens.pos++;
return token;
}
/* Note: parser_context_{peek|pop}_token do not increment the
* token object's refcount. In both cases the references will continue
* to be tracked and cleaned up in parser_context_free(), so do not
* attempt to free the token object.
*/
static QObject *parser_context_peek_token(JSONParserContext *ctxt)
{
QObject *token;
g_assert(ctxt->tokens.pos < ctxt->tokens.count);
token = ctxt->tokens.buf[ctxt->tokens.pos];
return token;
}
static JSONParserContext parser_context_save(JSONParserContext *ctxt)
{
JSONParserContext saved_ctxt = {0};
saved_ctxt.tokens.pos = ctxt->tokens.pos;
saved_ctxt.tokens.count = ctxt->tokens.count;
saved_ctxt.tokens.buf = ctxt->tokens.buf;
return saved_ctxt;
}
static void parser_context_restore(JSONParserContext *ctxt,
JSONParserContext saved_ctxt)
{
ctxt->tokens.pos = saved_ctxt.tokens.pos;
ctxt->tokens.count = saved_ctxt.tokens.count;
ctxt->tokens.buf = saved_ctxt.tokens.buf;
}
static void tokens_append_from_iter(QObject *obj, void *opaque)
{
JSONParserContext *ctxt = opaque;
g_assert(ctxt->tokens.pos < ctxt->tokens.count);
ctxt->tokens.buf[ctxt->tokens.pos++] = obj;
qobject_incref(obj);
}
static JSONParserContext *parser_context_new(QList *tokens)
{
JSONParserContext *ctxt;
size_t count;
if (!tokens) {
return NULL;
}
count = qlist_size(tokens);
if (count == 0) {
return NULL;
}
ctxt = g_malloc0(sizeof(JSONParserContext));
ctxt->tokens.pos = 0;
ctxt->tokens.count = count;
ctxt->tokens.buf = g_malloc(count * sizeof(QObject *));
qlist_iter(tokens, tokens_append_from_iter, ctxt);
ctxt->tokens.pos = 0;
return ctxt;
}
/* to support error propagation, ctxt->err must be freed separately */
static void parser_context_free(JSONParserContext *ctxt)
{
int i;
if (ctxt) {
for (i = 0; i < ctxt->tokens.count; i++) {
qobject_decref(ctxt->tokens.buf[i]);
}
g_free(ctxt->tokens.buf);
g_free(ctxt);
}
}
/** /**
* Parsing rules * Parsing rules
*/ */
static int parse_pair(JSONParserContext *ctxt, QDict *dict, QList **tokens, va_list *ap) static int parse_pair(JSONParserContext *ctxt, QDict *dict, va_list *ap)
{ {
QObject *key = NULL, *token = NULL, *value, *peek; QObject *key = NULL, *token = NULL, *value, *peek;
QList *working = qlist_copy(*tokens); JSONParserContext saved_ctxt = parser_context_save(ctxt);
peek = qlist_peek(working); peek = parser_context_peek_token(ctxt);
if (peek == NULL) { if (peek == NULL) {
parse_error(ctxt, NULL, "premature EOI"); parse_error(ctxt, NULL, "premature EOI");
goto out; goto out;
} }
key = parse_value(ctxt, &working, ap); key = parse_value(ctxt, ap);
if (!key || qobject_type(key) != QTYPE_QSTRING) { if (!key || qobject_type(key) != QTYPE_QSTRING) {
parse_error(ctxt, peek, "key is not a string in object"); parse_error(ctxt, peek, "key is not a string in object");
goto out; goto out;
} }
token = qlist_pop(working); token = parser_context_pop_token(ctxt);
if (token == NULL) { if (token == NULL) {
parse_error(ctxt, NULL, "premature EOI"); parse_error(ctxt, NULL, "premature EOI");
goto out; goto out;
@ -301,7 +390,7 @@ static int parse_pair(JSONParserContext *ctxt, QDict *dict, QList **tokens, va_l
goto out; goto out;
} }
value = parse_value(ctxt, &working, ap); value = parse_value(ctxt, ap);
if (value == NULL) { if (value == NULL) {
parse_error(ctxt, token, "Missing value in dict"); parse_error(ctxt, token, "Missing value in dict");
goto out; goto out;
@ -309,28 +398,24 @@ static int parse_pair(JSONParserContext *ctxt, QDict *dict, QList **tokens, va_l
qdict_put_obj(dict, qstring_get_str(qobject_to_qstring(key)), value); qdict_put_obj(dict, qstring_get_str(qobject_to_qstring(key)), value);
qobject_decref(token);
qobject_decref(key); qobject_decref(key);
QDECREF(*tokens);
*tokens = working;
return 0; return 0;
out: out:
qobject_decref(token); parser_context_restore(ctxt, saved_ctxt);
qobject_decref(key); qobject_decref(key);
QDECREF(working);
return -1; return -1;
} }
static QObject *parse_object(JSONParserContext *ctxt, QList **tokens, va_list *ap) static QObject *parse_object(JSONParserContext *ctxt, va_list *ap)
{ {
QDict *dict = NULL; QDict *dict = NULL;
QObject *token, *peek; QObject *token, *peek;
QList *working = qlist_copy(*tokens); JSONParserContext saved_ctxt = parser_context_save(ctxt);
token = qlist_pop(working); token = parser_context_pop_token(ctxt);
if (token == NULL) { if (token == NULL) {
goto out; goto out;
} }
@ -338,23 +423,22 @@ static QObject *parse_object(JSONParserContext *ctxt, QList **tokens, va_list *a
if (!token_is_operator(token, '{')) { if (!token_is_operator(token, '{')) {
goto out; goto out;
} }
qobject_decref(token);
token = NULL; token = NULL;
dict = qdict_new(); dict = qdict_new();
peek = qlist_peek(working); peek = parser_context_peek_token(ctxt);
if (peek == NULL) { if (peek == NULL) {
parse_error(ctxt, NULL, "premature EOI"); parse_error(ctxt, NULL, "premature EOI");
goto out; goto out;
} }
if (!token_is_operator(peek, '}')) { if (!token_is_operator(peek, '}')) {
if (parse_pair(ctxt, dict, &working, ap) == -1) { if (parse_pair(ctxt, dict, ap) == -1) {
goto out; goto out;
} }
token = qlist_pop(working); token = parser_context_pop_token(ctxt);
if (token == NULL) { if (token == NULL) {
parse_error(ctxt, NULL, "premature EOI"); parse_error(ctxt, NULL, "premature EOI");
goto out; goto out;
@ -365,59 +449,52 @@ static QObject *parse_object(JSONParserContext *ctxt, QList **tokens, va_list *a
parse_error(ctxt, token, "expected separator in dict"); parse_error(ctxt, token, "expected separator in dict");
goto out; goto out;
} }
qobject_decref(token);
token = NULL; token = NULL;
if (parse_pair(ctxt, dict, &working, ap) == -1) { if (parse_pair(ctxt, dict, ap) == -1) {
goto out; goto out;
} }
token = qlist_pop(working); token = parser_context_pop_token(ctxt);
if (token == NULL) { if (token == NULL) {
parse_error(ctxt, NULL, "premature EOI"); parse_error(ctxt, NULL, "premature EOI");
goto out; goto out;
} }
} }
qobject_decref(token);
token = NULL; token = NULL;
} else { } else {
token = qlist_pop(working); token = parser_context_pop_token(ctxt);
qobject_decref(token);
token = NULL; token = NULL;
} }
QDECREF(*tokens);
*tokens = working;
return QOBJECT(dict); return QOBJECT(dict);
out: out:
qobject_decref(token); parser_context_restore(ctxt, saved_ctxt);
QDECREF(working);
QDECREF(dict); QDECREF(dict);
return NULL; return NULL;
} }
static QObject *parse_array(JSONParserContext *ctxt, QList **tokens, va_list *ap) static QObject *parse_array(JSONParserContext *ctxt, va_list *ap)
{ {
QList *list = NULL; QList *list = NULL;
QObject *token, *peek; QObject *token, *peek;
QList *working = qlist_copy(*tokens); JSONParserContext saved_ctxt = parser_context_save(ctxt);
token = qlist_pop(working); token = parser_context_pop_token(ctxt);
if (token == NULL) { if (token == NULL) {
goto out; goto out;
} }
if (!token_is_operator(token, '[')) { if (!token_is_operator(token, '[')) {
token = NULL;
goto out; goto out;
} }
qobject_decref(token);
token = NULL; token = NULL;
list = qlist_new(); list = qlist_new();
peek = qlist_peek(working); peek = parser_context_peek_token(ctxt);
if (peek == NULL) { if (peek == NULL) {
parse_error(ctxt, NULL, "premature EOI"); parse_error(ctxt, NULL, "premature EOI");
goto out; goto out;
@ -426,7 +503,7 @@ static QObject *parse_array(JSONParserContext *ctxt, QList **tokens, va_list *ap
if (!token_is_operator(peek, ']')) { if (!token_is_operator(peek, ']')) {
QObject *obj; QObject *obj;
obj = parse_value(ctxt, &working, ap); obj = parse_value(ctxt, ap);
if (obj == NULL) { if (obj == NULL) {
parse_error(ctxt, token, "expecting value"); parse_error(ctxt, token, "expecting value");
goto out; goto out;
@ -434,7 +511,7 @@ static QObject *parse_array(JSONParserContext *ctxt, QList **tokens, va_list *ap
qlist_append_obj(list, obj); qlist_append_obj(list, obj);
token = qlist_pop(working); token = parser_context_pop_token(ctxt);
if (token == NULL) { if (token == NULL) {
parse_error(ctxt, NULL, "premature EOI"); parse_error(ctxt, NULL, "premature EOI");
goto out; goto out;
@ -446,10 +523,9 @@ static QObject *parse_array(JSONParserContext *ctxt, QList **tokens, va_list *ap
goto out; goto out;
} }
qobject_decref(token);
token = NULL; token = NULL;
obj = parse_value(ctxt, &working, ap); obj = parse_value(ctxt, ap);
if (obj == NULL) { if (obj == NULL) {
parse_error(ctxt, token, "expecting value"); parse_error(ctxt, token, "expecting value");
goto out; goto out;
@ -457,39 +533,33 @@ static QObject *parse_array(JSONParserContext *ctxt, QList **tokens, va_list *ap
qlist_append_obj(list, obj); qlist_append_obj(list, obj);
token = qlist_pop(working); token = parser_context_pop_token(ctxt);
if (token == NULL) { if (token == NULL) {
parse_error(ctxt, NULL, "premature EOI"); parse_error(ctxt, NULL, "premature EOI");
goto out; goto out;
} }
} }
qobject_decref(token);
token = NULL; token = NULL;
} else { } else {
token = qlist_pop(working); token = parser_context_pop_token(ctxt);
qobject_decref(token);
token = NULL; token = NULL;
} }
QDECREF(*tokens);
*tokens = working;
return QOBJECT(list); return QOBJECT(list);
out: out:
qobject_decref(token); parser_context_restore(ctxt, saved_ctxt);
QDECREF(working);
QDECREF(list); QDECREF(list);
return NULL; return NULL;
} }
static QObject *parse_keyword(JSONParserContext *ctxt, QList **tokens) static QObject *parse_keyword(JSONParserContext *ctxt)
{ {
QObject *token, *ret; QObject *token, *ret;
QList *working = qlist_copy(*tokens); JSONParserContext saved_ctxt = parser_context_save(ctxt);
token = qlist_pop(working); token = parser_context_pop_token(ctxt);
if (token == NULL) { if (token == NULL) {
goto out; goto out;
} }
@ -507,29 +577,24 @@ static QObject *parse_keyword(JSONParserContext *ctxt, QList **tokens)
goto out; goto out;
} }
qobject_decref(token);
QDECREF(*tokens);
*tokens = working;
return ret; return ret;
out: out:
qobject_decref(token); parser_context_restore(ctxt, saved_ctxt);
QDECREF(working);
return NULL; return NULL;
} }
static QObject *parse_escape(JSONParserContext *ctxt, QList **tokens, va_list *ap) static QObject *parse_escape(JSONParserContext *ctxt, va_list *ap)
{ {
QObject *token = NULL, *obj; QObject *token = NULL, *obj;
QList *working = qlist_copy(*tokens); JSONParserContext saved_ctxt = parser_context_save(ctxt);
if (ap == NULL) { if (ap == NULL) {
goto out; goto out;
} }
token = qlist_pop(working); token = parser_context_pop_token(ctxt);
if (token == NULL) { if (token == NULL) {
goto out; goto out;
} }
@ -553,25 +618,20 @@ static QObject *parse_escape(JSONParserContext *ctxt, QList **tokens, va_list *a
goto out; goto out;
} }
qobject_decref(token);
QDECREF(*tokens);
*tokens = working;
return obj; return obj;
out: out:
qobject_decref(token); parser_context_restore(ctxt, saved_ctxt);
QDECREF(working);
return NULL; return NULL;
} }
static QObject *parse_literal(JSONParserContext *ctxt, QList **tokens) static QObject *parse_literal(JSONParserContext *ctxt)
{ {
QObject *token, *obj; QObject *token, *obj;
QList *working = qlist_copy(*tokens); JSONParserContext saved_ctxt = parser_context_save(ctxt);
token = qlist_pop(working); token = parser_context_pop_token(ctxt);
if (token == NULL) { if (token == NULL) {
goto out; goto out;
} }
@ -591,35 +651,30 @@ static QObject *parse_literal(JSONParserContext *ctxt, QList **tokens)
goto out; goto out;
} }
qobject_decref(token);
QDECREF(*tokens);
*tokens = working;
return obj; return obj;
out: out:
qobject_decref(token); parser_context_restore(ctxt, saved_ctxt);
QDECREF(working);
return NULL; return NULL;
} }
static QObject *parse_value(JSONParserContext *ctxt, QList **tokens, va_list *ap) static QObject *parse_value(JSONParserContext *ctxt, va_list *ap)
{ {
QObject *obj; QObject *obj;
obj = parse_object(ctxt, tokens, ap); obj = parse_object(ctxt, ap);
if (obj == NULL) { if (obj == NULL) {
obj = parse_array(ctxt, tokens, ap); obj = parse_array(ctxt, ap);
} }
if (obj == NULL) { if (obj == NULL) {
obj = parse_escape(ctxt, tokens, ap); obj = parse_escape(ctxt, ap);
} }
if (obj == NULL) { if (obj == NULL) {
obj = parse_keyword(ctxt, tokens); obj = parse_keyword(ctxt);
} }
if (obj == NULL) { if (obj == NULL) {
obj = parse_literal(ctxt, tokens); obj = parse_literal(ctxt);
} }
return obj; return obj;
@ -632,19 +687,18 @@ QObject *json_parser_parse(QList *tokens, va_list *ap)
QObject *json_parser_parse_err(QList *tokens, va_list *ap, Error **errp) QObject *json_parser_parse_err(QList *tokens, va_list *ap, Error **errp)
{ {
JSONParserContext ctxt = {}; JSONParserContext *ctxt = parser_context_new(tokens);
QList *working;
QObject *result; QObject *result;
if (!tokens) { if (!ctxt) {
return NULL; return NULL;
} }
working = qlist_copy(tokens);
result = parse_value(&ctxt, &working, ap);
QDECREF(working); result = parse_value(ctxt, ap);
error_propagate(errp, ctxt.err); error_propagate(errp, ctxt->err);
parser_context_free(ctxt);
return result; return result;
} }