OpenE2K
/
gcc
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

[C++ PATCH] simplify deferred parsing lexer 

https://gcc.gnu.org/ml/gcc-patches/2019-10/msg01962.html

We use an eof_token global variable as a sentinel on a deferred parse
(such as in-class function definitions, or default args). This
complicates retrieving the next token in certain places.

As such deferred parses always nest properly and completely before
resuming the outer lexer, we can simply morph the token after the
deferred buffer into a CPP_EOF token and restore it afterwards. I
finally got around to implementing it with this patch.

One complication is that we have to change the discriminator for when
the token's value is a tree. We can't look at the token's type because
it might have been overwritten. I add a bool flag to the token
(there's several spare bits), and use that. This does simplify the
discriminator because we just check a single bit, rather than a set of
token types.

	* parser.h (struct cp_token): Drop {ENUM,BOOL}_BITFIELD C-ism.
	Add tree_check_p flag, use as nested union discriminator.
	(struct cp_lexer): Add saved_type & saved_keyword fields.
	* parser.c (eof_token): Delete.
	(cp_lexer_new_main): Always init last_token to last token of
	buffer.
	(cp_lexer_new_from_tokens): Overlay EOF token at end of range.
	(cp_lexer_destroy): Restore token under the EOF.
	(cp_lexer_previous_token_position): No check for eof_token here.
	(cp_lexer_get_preprocessor_token): Clear tree_check_p.
	(cp_lexer_peek_nth_token): Check CPP_EOF not eof_token.
	(cp_lexer_consume_token): Assert not CPP_EOF, no check for
	eof_token.
	(cp_lexer_purge_token): Likewise.
	(cp_lexer_purge_tokens_after): No check for EOF token.
	(cp_parser_nested_name_specifier, cp_parser_decltype)
	(cp_parser_template_id): Set tree_check_p.

From-SVN: r277514
This commit is contained in:
Nathan Sidwell 2019-10-28 12:35:39 +00:00 committed by Nathan Sidwell
parent e0c4f7fbd6
commit 14c835a01c
3 changed files with 63 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
gcc/cp/ChangeLog
View File

@ -1,3 +1,23 @@
2019-10-28 Nathan Sidwell <nathan@acm.org>
* parser.h (struct cp_token): Drop {ENUM,BOOL}_BITFIELD C-ism.
Add tree_check_p flag, use as nested union discriminator.
(struct cp_lexer): Add saved_type & saved_keyword fields.
* parser.c (eof_token): Delete.
(cp_lexer_new_main): Always init last_token to last token of
buffer.
(cp_lexer_new_from_tokens): Overlay EOF token at end of range.
(cp_lexer_destroy): Restore token under the EOF.
(cp_lexer_previous_token_position): No check for eof_token here.
(cp_lexer_get_preprocessor_token): Clear tree_check_p.
(cp_lexer_peek_nth_token): Check CPP_EOF not eof_token.
(cp_lexer_consume_token): Assert not CPP_EOF, no check for
eof_token.
(cp_lexer_purge_token): Likewise.
(cp_lexer_purge_tokens_after): No check for EOF token.
(cp_parser_nested_name_specifier, cp_parser_decltype)
(cp_parser_template_id): Set tree_check_p.
2019-10-24 Jakub Jelinek <jakub@redhat.com>
* decl2.c (cplus_decl_attributes): Add "omp declare target block"

73
gcc/cp/parser.c
View File

@ -52,11 +52,6 @@ along with GCC; see the file COPYING3. If not see
/* The cp_lexer_* routines mediate between the lexer proper (in libcpp
and c-lex.c) and the C++ parser. */
static cp_token eof_token =
{
CPP_EOF, RID_MAX, 0, false, false, false, 0, { NULL }
};
/* The various kinds of non integral constant we encounter. */
enum non_integral_constant {
NIC_NONE,
@ -660,12 +655,10 @@ cp_lexer_new_main (void)
vec_safe_push (lexer->buffer, token);
}
lexer->last_token = lexer->buffer->address ()
lexer->next_token = lexer->buffer->address ();
lexer->last_token = lexer->next_token
+ lexer->buffer->length ()
- 1;
lexer->next_token = lexer->buffer->length ()
? lexer->buffer->address ()
: &eof_token;
/* Subsequent preprocessor diagnostics should use compiler
diagnostic functions to get the compiler source location. */
@ -687,7 +680,14 @@ cp_lexer_new_from_tokens (cp_token_cache *cache)
/* We do not own the buffer. */
lexer->buffer = NULL;
lexer->next_token = first == last ? &eof_token : first;
/* Insert an EOF token. */
lexer->saved_type = last->type;
lexer->saved_keyword = last->keyword;
last->type = CPP_EOF;
last->keyword = RID_MAX;
lexer->next_token = first;
lexer->last_token = last;
lexer->saved_tokens.create (CP_SAVED_TOKEN_STACK);
@ -704,7 +704,14 @@ cp_lexer_new_from_tokens (cp_token_cache *cache)
static void
cp_lexer_destroy (cp_lexer *lexer)
{
vec_free (lexer->buffer);
if (lexer->buffer)
vec_free (lexer->buffer);
else
{
/* Restore the token we overwrite with EOF. */
lexer->last_token->type = lexer->saved_type;
lexer->last_token->keyword = lexer->saved_keyword;
}
lexer->saved_tokens.release ();
ggc_free (lexer);
}
@ -731,8 +738,6 @@ cp_lexer_debugging_p (cp_lexer *lexer)
static inline cp_token_position
cp_lexer_token_position (cp_lexer *lexer, bool previous_p)
{
gcc_assert (!previous_p || lexer->next_token != &eof_token);
return lexer->next_token - previous_p;
}
@ -751,10 +756,7 @@ cp_lexer_set_token_position (cp_lexer *lexer, cp_token_position pos)
static inline cp_token_position
cp_lexer_previous_token_position (cp_lexer *lexer)
{
if (lexer->next_token == &eof_token)
return lexer->last_token - 1;
else
return cp_lexer_token_position (lexer, true);
return cp_lexer_token_position (lexer, true);
}
static inline cp_token *
@ -807,6 +809,7 @@ cp_lexer_get_preprocessor_token (cp_lexer *lexer, cp_token *token)
token->keyword = RID_MAX;
token->purged_p = false;
token->error_reported = false;
token->tree_check_p = false;
/* On some systems, some header files are surrounded by an
implicit extern "C" block. Set a flag in the token if it
@ -1082,16 +1085,9 @@ cp_lexer_peek_nth_token (cp_lexer* lexer, size_t n)
--n;
token = lexer->next_token;
gcc_assert (!n || token != &eof_token);
while (n != 0)
while (n && token->type != CPP_EOF)
{
++token;
if (token == lexer->last_token)
{
token = &eof_token;
break;
}
if (!token->purged_p)
--n;
}
@ -1113,18 +1109,12 @@ cp_lexer_consume_token (cp_lexer* lexer)
{
cp_token *token = lexer->next_token;
gcc_assert (token != &eof_token);
gcc_assert (!lexer->in_pragma || token->type != CPP_PRAGMA_EOL);
do
{
gcc_assert (token->type != CPP_EOF);
lexer->next_token++;
if (lexer->next_token == lexer->last_token)
{
lexer->next_token = &eof_token;
break;
}
}
while (lexer->next_token->purged_p);
@ -1150,21 +1140,14 @@ cp_lexer_purge_token (cp_lexer *lexer)
{
cp_token *tok = lexer->next_token;
gcc_assert (tok != &eof_token);
gcc_assert (tok->type != CPP_EOF);
tok->purged_p = true;
tok->location = UNKNOWN_LOCATION;
tok->u.value = NULL_TREE;
tok->keyword = RID_MAX;
do
{
tok++;
if (tok == lexer->last_token)
{
tok = &eof_token;
break;
}
}
tok++;
while (tok->purged_p);
lexer->next_token = tok;
}
@ -1178,12 +1161,9 @@ cp_lexer_purge_tokens_after (cp_lexer *lexer, cp_token *tok)
{
cp_token *peek = lexer->next_token;
if (peek == &eof_token)
peek = lexer->last_token;
gcc_assert (tok < peek);
for ( tok += 1; tok != peek; tok += 1)
for (tok++; tok != peek; tok++)
{
tok->purged_p = true;
tok->location = UNKNOWN_LOCATION;
@ -6616,6 +6596,7 @@ cp_parser_nested_name_specifier_opt (cp_parser *parser,
/* Retrieve any deferred checks. Do not pop this access checks yet
so the memory will not be reclaimed during token replacing below. */
token->u.tree_check_value = ggc_cleared_alloc<struct tree_check> ();
token->tree_check_p = true;
token->u.tree_check_value->value = parser->scope;
token->u.tree_check_value->checks = get_deferred_access_checks ();
token->u.tree_check_value->qualifying_scope =
@ -14801,6 +14782,7 @@ cp_parser_decltype (cp_parser *parser)
it again. */
start_token->type = CPP_DECLTYPE;
start_token->u.tree_check_value = ggc_cleared_alloc<struct tree_check> ();
start_token->tree_check_p = true;
start_token->u.tree_check_value->value = expr;
start_token->u.tree_check_value->checks = get_deferred_access_checks ();
start_token->keyword = RID_MAX;
@ -16588,6 +16570,7 @@ cp_parser_template_id (cp_parser *parser,
/* Retrieve any deferred checks. Do not pop this access checks yet
so the memory will not be reclaimed during token replacing below. */
token->u.tree_check_value = ggc_cleared_alloc<struct tree_check> ();
token->tree_check_p = true;
token->u.tree_check_value->value = template_id;
token->u.tree_check_value->checks = get_deferred_access_checks ();
token->keyword = RID_MAX;

26
gcc/cp/parser.h
View File

@ -41,34 +41,34 @@ struct GTY(()) tree_check {
struct GTY (()) cp_token {
/* The kind of token. */
ENUM_BITFIELD (cpp_ttype) type : 8;
enum cpp_ttype type : 8;
/* If this token is a keyword, this value indicates which keyword.
Otherwise, this value is RID_MAX. */
ENUM_BITFIELD (rid) keyword : 8;
enum rid keyword : 8;
/* Token flags. */
unsigned char flags;
/* True if this token is from a context where it is implicitly extern "C" */
BOOL_BITFIELD implicit_extern_c : 1;
bool implicit_extern_c : 1;
/* True if an error has already been reported for this token, such as a
CPP_NAME token that is not a keyword (i.e., for which KEYWORD is
RID_MAX) iff this name was looked up and found to be ambiguous. */
BOOL_BITFIELD error_reported : 1;
bool error_reported : 1;
/* True for a token that has been purged. If a token is purged,
it is no longer a valid token and it should be considered
deleted. */
BOOL_BITFIELD purged_p : 1;
/* 5 unused bits. */
bool purged_p : 1;
bool tree_check_p : 1;
/* 4 unused bits. */
/* The location at which this token was found. */
location_t location;
/* The value associated with this token, if any. */
union cp_token_value {
/* Used for compound tokens such as CPP_NESTED_NAME_SPECIFIER. */
struct tree_check* GTY((tag ("1"))) tree_check_value;
struct tree_check* GTY((tag ("true"))) tree_check_value;
/* Use for all other tokens. */
tree GTY((tag ("0"))) value;
} GTY((desc ("(%1.type == CPP_TEMPLATE_ID)"
"|| (%1.type == CPP_NESTED_NAME_SPECIFIER)"
"|| (%1.type == CPP_DECLTYPE)"))) u;
tree GTY((tag ("false"))) value;
} GTY((desc ("%1.tree_check_p"))) u;
};
@ -99,6 +99,10 @@ struct GTY (()) cp_lexer {
tokens. */
vec<cp_token_position> GTY ((skip)) saved_tokens;
/* Saved pieces of end token we replaced with the eof token. */
enum cpp_ttype saved_type : 8;
enum rid saved_keyword : 8;
/* The next lexer in a linked list of lexers. */
struct cp_lexer *next;