qjson: store tokens in a GQueue

Even though we still have the "streamer" concept, the tokens can now
be deleted as they are read.  While doing so convert from QList to
GQueue, since the next step will make tokens not a QObject and we
will have to do the conversion anyway.

Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
Message-Id: <1448300659-23559-4-git-send-email-pbonzini@redhat.com>
Signed-off-by: Markus Armbruster <armbru@redhat.com>
Reviewed-by: Eric Blake <eblake@redhat.com>
This commit is contained in:
Paolo Bonzini 2015-11-25 22:23:31 +01:00 committed by Markus Armbruster
parent d538b25543
commit 95385fe9ac
8 changed files with 45 additions and 65 deletions

View File

@ -18,7 +18,7 @@
#include "qapi/qmp/qlist.h" #include "qapi/qmp/qlist.h"
#include "qapi/error.h" #include "qapi/error.h"
QObject *json_parser_parse(QList *tokens, va_list *ap); QObject *json_parser_parse(GQueue *tokens, va_list *ap);
QObject *json_parser_parse_err(QList *tokens, va_list *ap, Error **errp); QObject *json_parser_parse_err(GQueue *tokens, va_list *ap, Error **errp);
#endif #endif

View File

@ -15,21 +15,21 @@
#define QEMU_JSON_STREAMER_H #define QEMU_JSON_STREAMER_H
#include <stdint.h> #include <stdint.h>
#include "qapi/qmp/qlist.h" #include "glib-compat.h"
#include "qapi/qmp/json-lexer.h" #include "qapi/qmp/json-lexer.h"
typedef struct JSONMessageParser typedef struct JSONMessageParser
{ {
void (*emit)(struct JSONMessageParser *parser, QList *tokens); void (*emit)(struct JSONMessageParser *parser, GQueue *tokens);
JSONLexer lexer; JSONLexer lexer;
int brace_count; int brace_count;
int bracket_count; int bracket_count;
QList *tokens; GQueue *tokens;
uint64_t token_size; uint64_t token_size;
} JSONMessageParser; } JSONMessageParser;
void json_message_parser_init(JSONMessageParser *parser, void json_message_parser_init(JSONMessageParser *parser,
void (*func)(JSONMessageParser *, QList *)); void (*func)(JSONMessageParser *, GQueue *));
int json_message_parser_feed(JSONMessageParser *parser, int json_message_parser_feed(JSONMessageParser *parser,
const char *buffer, size_t size); const char *buffer, size_t size);

View File

@ -3849,7 +3849,7 @@ static QDict *qmp_check_input_obj(QObject *input_obj, Error **errp)
return input_dict; return input_dict;
} }
static void handle_qmp_command(JSONMessageParser *parser, QList *tokens) static void handle_qmp_command(JSONMessageParser *parser, GQueue *tokens)
{ {
Error *local_err = NULL; Error *local_err = NULL;
QObject *obj, *data; QObject *obj, *data;

View File

@ -570,7 +570,7 @@ static void process_command(GAState *s, QDict *req)
} }
/* handle requests/control events coming in over the channel */ /* handle requests/control events coming in over the channel */
static void process_event(JSONMessageParser *parser, QList *tokens) static void process_event(JSONMessageParser *parser, GQueue *tokens)
{ {
GAState *s = container_of(parser, GAState, parser); GAState *s = container_of(parser, GAState, parser);
QDict *qdict; QDict *qdict;

View File

@ -26,11 +26,8 @@
typedef struct JSONParserContext typedef struct JSONParserContext
{ {
Error *err; Error *err;
struct { QObject *current;
QObject **buf; GQueue *buf;
size_t pos;
size_t count;
} tokens;
} JSONParserContext; } JSONParserContext;
#define BUG_ON(cond) assert(!(cond)) #define BUG_ON(cond) assert(!(cond))
@ -243,56 +240,34 @@ out:
return NULL; return NULL;
} }
/* Note: unless the token object returned by parser_context_peek_token
* or parser_context_pop_token is explicitly incref'd, it will be
* deleted as soon as parser_context_pop_token is called again.
*/
static QObject *parser_context_pop_token(JSONParserContext *ctxt) static QObject *parser_context_pop_token(JSONParserContext *ctxt)
{ {
QObject *token; qobject_decref(ctxt->current);
g_assert(ctxt->tokens.pos < ctxt->tokens.count); assert(!g_queue_is_empty(ctxt->buf));
token = ctxt->tokens.buf[ctxt->tokens.pos]; ctxt->current = g_queue_pop_head(ctxt->buf);
ctxt->tokens.pos++; return ctxt->current;
return token;
} }
/* Note: parser_context_{peek|pop}_token do not increment the
* token object's refcount. In both cases the references will continue
* to be tracked and cleaned up in parser_context_free(), so do not
* attempt to free the token object.
*/
static QObject *parser_context_peek_token(JSONParserContext *ctxt) static QObject *parser_context_peek_token(JSONParserContext *ctxt)
{ {
QObject *token; assert(!g_queue_is_empty(ctxt->buf));
g_assert(ctxt->tokens.pos < ctxt->tokens.count); return g_queue_peek_head(ctxt->buf);
token = ctxt->tokens.buf[ctxt->tokens.pos];
return token;
} }
static void tokens_append_from_iter(QObject *obj, void *opaque) static JSONParserContext *parser_context_new(GQueue *tokens)
{
JSONParserContext *ctxt = opaque;
g_assert(ctxt->tokens.pos < ctxt->tokens.count);
ctxt->tokens.buf[ctxt->tokens.pos++] = obj;
qobject_incref(obj);
}
static JSONParserContext *parser_context_new(QList *tokens)
{ {
JSONParserContext *ctxt; JSONParserContext *ctxt;
size_t count;
if (!tokens) { if (!tokens) {
return NULL; return NULL;
} }
count = qlist_size(tokens);
if (count == 0) {
return NULL;
}
ctxt = g_malloc0(sizeof(JSONParserContext)); ctxt = g_malloc0(sizeof(JSONParserContext));
ctxt->tokens.pos = 0; ctxt->buf = tokens;
ctxt->tokens.count = count;
ctxt->tokens.buf = g_malloc(count * sizeof(QObject *));
qlist_iter(tokens, tokens_append_from_iter, ctxt);
ctxt->tokens.pos = 0;
return ctxt; return ctxt;
} }
@ -300,12 +275,12 @@ static JSONParserContext *parser_context_new(QList *tokens)
/* to support error propagation, ctxt->err must be freed separately */ /* to support error propagation, ctxt->err must be freed separately */
static void parser_context_free(JSONParserContext *ctxt) static void parser_context_free(JSONParserContext *ctxt)
{ {
int i;
if (ctxt) { if (ctxt) {
for (i = 0; i < ctxt->tokens.count; i++) { while (!g_queue_is_empty(ctxt->buf)) {
qobject_decref(ctxt->tokens.buf[i]); parser_context_pop_token(ctxt);
} }
g_free(ctxt->tokens.buf); qobject_decref(ctxt->current);
g_queue_free(ctxt->buf);
g_free(ctxt); g_free(ctxt);
} }
} }
@ -598,12 +573,12 @@ static QObject *parse_value(JSONParserContext *ctxt, va_list *ap)
} }
} }
QObject *json_parser_parse(QList *tokens, va_list *ap) QObject *json_parser_parse(GQueue *tokens, va_list *ap)
{ {
return json_parser_parse_err(tokens, ap, NULL); return json_parser_parse_err(tokens, ap, NULL);
} }
QObject *json_parser_parse_err(QList *tokens, va_list *ap, Error **errp) QObject *json_parser_parse_err(GQueue *tokens, va_list *ap, Error **errp)
{ {
JSONParserContext *ctxt = parser_context_new(tokens); JSONParserContext *ctxt = parser_context_new(tokens);
QObject *result; QObject *result;

View File

@ -22,6 +22,14 @@
#define MAX_TOKEN_SIZE (64ULL << 20) #define MAX_TOKEN_SIZE (64ULL << 20)
#define MAX_NESTING (1ULL << 10) #define MAX_NESTING (1ULL << 10)
static void json_message_free_tokens(JSONMessageParser *parser)
{
if (parser->tokens) {
g_queue_free(parser->tokens);
parser->tokens = NULL;
}
}
static void json_message_process_token(JSONLexer *lexer, GString *input, static void json_message_process_token(JSONLexer *lexer, GString *input,
JSONTokenType type, int x, int y) JSONTokenType type, int x, int y)
{ {
@ -53,7 +61,7 @@ static void json_message_process_token(JSONLexer *lexer, GString *input,
parser->token_size += input->len; parser->token_size += input->len;
qlist_append(parser->tokens, dict); g_queue_push_tail(parser->tokens, dict);
if (type == JSON_ERROR) { if (type == JSON_ERROR) {
goto out_emit_bad; goto out_emit_bad;
@ -77,27 +85,24 @@ out_emit_bad:
* Clear out token list and tell the parser to emit an error * Clear out token list and tell the parser to emit an error
* indication by passing it a NULL list * indication by passing it a NULL list
*/ */
QDECREF(parser->tokens); json_message_free_tokens(parser);
parser->tokens = NULL;
out_emit: out_emit:
/* send current list of tokens to parser and reset tokenizer */ /* send current list of tokens to parser and reset tokenizer */
parser->brace_count = 0; parser->brace_count = 0;
parser->bracket_count = 0; parser->bracket_count = 0;
/* parser->emit takes ownership of parser->tokens. */
parser->emit(parser, parser->tokens); parser->emit(parser, parser->tokens);
if (parser->tokens) { parser->tokens = g_queue_new();
QDECREF(parser->tokens);
}
parser->tokens = qlist_new();
parser->token_size = 0; parser->token_size = 0;
} }
void json_message_parser_init(JSONMessageParser *parser, void json_message_parser_init(JSONMessageParser *parser,
void (*func)(JSONMessageParser *, QList *)) void (*func)(JSONMessageParser *, GQueue *))
{ {
parser->emit = func; parser->emit = func;
parser->brace_count = 0; parser->brace_count = 0;
parser->bracket_count = 0; parser->bracket_count = 0;
parser->tokens = qlist_new(); parser->tokens = g_queue_new();
parser->token_size = 0; parser->token_size = 0;
json_lexer_init(&parser->lexer, json_message_process_token); json_lexer_init(&parser->lexer, json_message_process_token);
@ -117,5 +122,5 @@ int json_message_parser_flush(JSONMessageParser *parser)
void json_message_parser_destroy(JSONMessageParser *parser) void json_message_parser_destroy(JSONMessageParser *parser)
{ {
json_lexer_destroy(&parser->lexer); json_lexer_destroy(&parser->lexer);
QDECREF(parser->tokens); json_message_free_tokens(parser);
} }

View File

@ -28,7 +28,7 @@ typedef struct JSONParsingState
QObject *result; QObject *result;
} JSONParsingState; } JSONParsingState;
static void parse_json(JSONMessageParser *parser, QList *tokens) static void parse_json(JSONMessageParser *parser, GQueue *tokens)
{ {
JSONParsingState *s = container_of(parser, JSONParsingState, parser); JSONParsingState *s = container_of(parser, JSONParsingState, parser);
s->result = json_parser_parse(tokens, s->ap); s->result = json_parser_parse(tokens, s->ap);

View File

@ -351,7 +351,7 @@ typedef struct {
QDict *response; QDict *response;
} QMPResponseParser; } QMPResponseParser;
static void qmp_response(JSONMessageParser *parser, QList *tokens) static void qmp_response(JSONMessageParser *parser, GQueue *tokens)
{ {
QMPResponseParser *qmp = container_of(parser, QMPResponseParser, parser); QMPResponseParser *qmp = container_of(parser, QMPResponseParser, parser);
QObject *obj; QObject *obj;