OpenE2K
/
gcc
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

macros: Add remaining context and improve parsing macro dispatch 

This allows us to expand macor invocations in more places, as macro
calls are not limited to statements or expressions. It is quite common
to use macros to abstract writing repetitive boilerplate for type
implementations, for example.
This commit is contained in:
Arthur Cohen 2022-03-14 17:08:07 +01:00
parent 1a14348afe
commit 1a2ef9cae9
8 changed files with 298 additions and 91 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

119
gcc/rust/ast/rust-ast.h
View File

@ -1511,18 +1511,45 @@ public:
EXPRESSION,
ITEM,
STMT,
EXTERN,
TRAIT,
IMPL,
};
private:
NodeType kind;
// FIXME make this a union
std::unique_ptr<Expr> expr;
std::unique_ptr<Item> item;
std::unique_ptr<Stmt> stmt;
std::unique_ptr<ExternalItem> external_item;
std::unique_ptr<TraitItem> trait_item;
std::unique_ptr<InherentImplItem> impl_item;
public:
SingleASTNode (std::unique_ptr<Expr> expr)
: kind (EXPRESSION), expr (std::move (expr)), item (nullptr), stmt (nullptr)
: kind (EXPRESSION), expr (std::move (expr))
{}
SingleASTNode (std::unique_ptr<Item> item)
: kind (ITEM), expr (nullptr), item (std::move (item)), stmt (nullptr)
: kind (ITEM), item (std::move (item))
{}
SingleASTNode (std::unique_ptr<Stmt> stmt)
: kind (STMT), expr (nullptr), item (nullptr), stmt (std::move (stmt))
: kind (STMT), stmt (std::move (stmt))
{}
SingleASTNode (std::unique_ptr<ExternalItem> item)
: kind (EXTERN), external_item (std::move (item))
{}
SingleASTNode (std::unique_ptr<TraitItem> item)
: kind (TRAIT), trait_item (std::move (item))
{}
SingleASTNode (std::unique_ptr<InherentImplItem> item)
: kind (IMPL), impl_item (std::move (item))
{}
SingleASTNode (SingleASTNode const &other)
@ -1541,6 +1568,18 @@ public:
case STMT:
stmt = other.stmt->clone_stmt ();
break;
case EXTERN:
external_item = other.external_item->clone_external_item ();
break;
case TRAIT:
trait_item = other.trait_item->clone_trait_item ();
break;
case IMPL:
impl_item = other.impl_item->clone_inherent_impl_item ();
break;
}
}
@ -1560,6 +1599,18 @@ public:
case STMT:
stmt = other.stmt->clone_stmt ();
break;
case EXTERN:
external_item = other.external_item->clone_external_item ();
break;
case TRAIT:
trait_item = other.trait_item->clone_trait_item ();
break;
case IMPL:
impl_item = other.impl_item->clone_inherent_impl_item ();
break;
}
return *this;
}
@ -1569,7 +1620,7 @@ public:
NodeType get_kind () const { return kind; }
std::unique_ptr<Expr> &get_inner ()
std::unique_ptr<Expr> &get_expr ()
{
rust_assert (kind == EXPRESSION);
return expr;
@ -1610,6 +1661,24 @@ public:
return std::move (item);
}
std::unique_ptr<TraitItem> take_trait_item ()
{
rust_assert (!is_error ());
return std::move (trait_item);
}
std::unique_ptr<ExternalItem> take_external_item ()
{
rust_assert (!is_error ());
return std::move (external_item);
}
std::unique_ptr<InherentImplItem> take_impl_item ()
{
rust_assert (!is_error ());
return std::move (impl_item);
}
void accept_vis (ASTVisitor &vis)
{
switch (kind)
@ -1625,6 +1694,18 @@ public:
case STMT:
stmt->accept_vis (vis);
break;
case EXTERN:
external_item->accept_vis (vis);
break;
case TRAIT:
trait_item->accept_vis (vis);
break;
case IMPL:
impl_item->accept_vis (vis);
break;
}
}
@ -1638,9 +1719,16 @@ public:
return item == nullptr;
case STMT:
return stmt == nullptr;
default:
return true;
case EXTERN:
return external_item == nullptr;
case TRAIT:
return trait_item == nullptr;
case IMPL:
return impl_item == nullptr;
}
gcc_unreachable ();
return true;
}
std::string as_string ()
@ -1653,18 +1741,17 @@ public:
return "Item: " + item->as_string ();
case STMT:
return "Stmt: " + stmt->as_string ();
default:
return "";
case EXTERN:
return "External Item: " + external_item->as_string ();
case TRAIT:
return "Trait Item: " + trait_item->as_string ();
case IMPL:
return "Impl Item: " + impl_item->as_string ();
}
gcc_unreachable ();
return "";
}
private:
NodeType kind;
// FIXME make this a union
std::unique_ptr<Expr> expr;
std::unique_ptr<Item> item;
std::unique_ptr<Stmt> stmt;
};
/* Basically, a "fragment" that can be incorporated into the AST, created as

6
gcc/rust/ast/rust-macro.h
View File

@ -460,6 +460,7 @@ class MacroInvocation : public TypeNoBounds,
public TraitItem,
public TraitImplItem,
public InherentImplItem,
public ExternalItem,
public ExprWithoutBlock
{
std::vector<Attribute> outer_attrs;
@ -537,6 +538,11 @@ protected:
return clone_macro_invocation_impl ();
}
MacroInvocation *clone_external_item_impl () const final override
{
return clone_macro_invocation_impl ();
}
/*virtual*/ MacroInvocation *clone_macro_invocation_impl () const
{
return new MacroInvocation (*this);

52
gcc/rust/expand/rust-attribute-visitor.cc
View File

@ -1114,8 +1114,6 @@ AttrVisitor::visit (AST::ClosureExprInner &expr)
void
AttrVisitor::visit (AST::BlockExpr &expr)
{
expander.push_context (MacroExpander::BLOCK);
// initial strip test based on outer attrs
expander.expand_cfg_attrs (expr.get_outer_attrs ());
if (expander.fails_cfg_with_expand (expr.get_outer_attrs ()))
@ -1135,31 +1133,13 @@ AttrVisitor::visit (AST::BlockExpr &expr)
return;
}
// strip all statements
auto &stmts = expr.get_statements ();
for (auto it = stmts.begin (); it != stmts.end ();)
{
auto &stmt = *it;
std::function<std::unique_ptr<AST::Stmt> (AST::SingleASTNode)> extractor
= [] (AST::SingleASTNode node) { return node.take_stmt (); };
stmt->accept_vis (*this);
expand_macro_children (MacroExpander::BLOCK, expr.get_statements (),
extractor);
auto fragment = expander.take_expanded_fragment (*this);
if (fragment.should_expand ())
{
// Remove the current expanded invocation
it = stmts.erase (it);
for (auto &node : fragment.get_nodes ())
{
it = stmts.insert (it, node.take_stmt ());
it++;
}
}
else if (stmt->is_marked_for_strip ())
it = stmts.erase (it);
else
it++;
}
expander.push_context (MacroExpander::BLOCK);
// strip tail expression if exists - can actually fully remove it
if (expr.has_tail_expr ())
@ -2489,8 +2469,11 @@ AttrVisitor::visit (AST::Trait &trait)
if (trait.has_where_clause ())
expand_where_clause (trait.get_where_clause ());
// strip trait items if required
expand_pointer_allow_strip (trait.get_trait_items ());
std::function<std::unique_ptr<AST::TraitItem> (AST::SingleASTNode)> extractor
= [] (AST::SingleASTNode node) { return node.take_trait_item (); };
expand_macro_children (MacroExpander::TRAIT, trait.get_trait_items (),
extractor);
}
void
AttrVisitor::visit (AST::InherentImpl &impl)
@ -2523,8 +2506,11 @@ AttrVisitor::visit (AST::InherentImpl &impl)
if (impl.has_where_clause ())
expand_where_clause (impl.get_where_clause ());
// strip inherent impl items if required
expand_pointer_allow_strip (impl.get_impl_items ());
std::function<std::unique_ptr<AST::InherentImplItem> (AST::SingleASTNode)>
extractor = [] (AST::SingleASTNode node) { return node.take_impl_item (); };
expand_macro_children (MacroExpander::IMPL, impl.get_impl_items (),
extractor);
}
void
AttrVisitor::visit (AST::TraitImpl &impl)
@ -2659,8 +2645,12 @@ AttrVisitor::visit (AST::ExternBlock &block)
return;
}
// strip external items if required
expand_pointer_allow_strip (block.get_extern_items ());
std::function<std::unique_ptr<AST::ExternalItem> (AST::SingleASTNode)>
extractor
= [] (AST::SingleASTNode node) { return node.take_external_item (); };
expand_macro_children (MacroExpander::EXTERN, block.get_extern_items (),
extractor);
}
// I don't think it would be possible to strip macros without expansion

57
gcc/rust/expand/rust-attribute-visitor.h
View File

@ -40,6 +40,54 @@ public:
void expand_trait_function_decl (AST::TraitFunctionDecl &decl);
void expand_trait_method_decl (AST::TraitMethodDecl &decl);
/**
* Expand a set of values, erasing them if they are marked for strip, and
* replacing them with expanded macro nodes if necessary.
* This function is slightly different from `expand_pointer_allow_strip` as
* it can only be called in certain expansion contexts - where macro
* invocations are allowed.
*
* @param ctx Context to use for macro expansion
* @param values Iterable reference over values to replace or erase
* @param extractor Function to call when replacing values with the content
* of an expanded AST node
*/
template <typename T, typename U>
void expand_macro_children (MacroExpander::ContextType ctx, T &values,
std::function<U (AST::SingleASTNode)> extractor)
{
expander.push_context (ctx);
for (auto it = values.begin (); it != values.end ();)
{
auto &value = *it;
// mark for stripping if required
value->accept_vis (*this);
auto fragment = expander.take_expanded_fragment (*this);
if (fragment.should_expand ())
{
it = values.erase (it);
for (auto &node : fragment.get_nodes ())
{
it = values.insert (it, extractor (node));
it++;
}
}
else if (value->is_marked_for_strip ())
{
it = values.erase (it);
}
else
{
++it;
}
}
expander.pop_context ();
}
template <typename T> void expand_pointer_allow_strip (T &values)
{
for (auto it = values.begin (); it != values.end ();)
@ -48,11 +96,14 @@ public:
// mark for stripping if required
value->accept_vis (*this);
if (value->is_marked_for_strip ())
it = values.erase (it);
{
it = values.erase (it);
}
else
++it;
{
++it;
}
}
}

139
gcc/rust/expand/rust-macro-expand.cc
View File

@ -786,7 +786,6 @@ parse_many (Parser<MacroInvocLexer> &parser, TokenId &delimiter,
std::function<AST::SingleASTNode ()> parse_fn)
{
std::vector<AST::SingleASTNode> nodes;
while (true)
{
if (parser.peek_current_token ()->get_id () == delimiter)
@ -814,6 +813,52 @@ transcribe_many_items (Parser<MacroInvocLexer> &parser, TokenId &delimiter)
});
}
/**
* Transcribe 0 or more external items from a macro invocation
*
* @param parser Parser to extract items from
* @param delimiter Id of the token on which parsing should stop
*/
static std::vector<AST::SingleASTNode>
transcribe_many_ext (Parser<MacroInvocLexer> &parser, TokenId &delimiter)
{
return parse_many (parser, delimiter, [&parser] () {
auto item = parser.parse_external_item ();
return AST::SingleASTNode (std::move (item));
});
}
/**
* Transcribe 0 or more trait items from a macro invocation
*
* @param parser Parser to extract items from
* @param delimiter Id of the token on which parsing should stop
*/
static std::vector<AST::SingleASTNode>
transcribe_many_trait_items (Parser<MacroInvocLexer> &parser,
TokenId &delimiter)
{
return parse_many (parser, delimiter, [&parser] () {
auto item = parser.parse_trait_item ();
return AST::SingleASTNode (std::move (item));
});
}
/**
* Transcribe 0 or more impl items from a macro invocation
*
* @param parser Parser to extract items from
* @param delimiter Id of the token on which parsing should stop
*/
static std::vector<AST::SingleASTNode>
transcribe_many_impl_items (Parser<MacroInvocLexer> &parser, TokenId &delimiter)
{
return parse_many (parser, delimiter, [&parser] () {
auto item = parser.parse_inherent_impl_item ();
return AST::SingleASTNode (std::move (item));
});
}
/**
* Transcribe 0 or more statements from a macro invocation
*
@ -845,6 +890,57 @@ transcribe_expression (Parser<MacroInvocLexer> &parser)
return {AST::SingleASTNode (std::move (expr))};
}
static std::vector<AST::SingleASTNode>
transcribe_on_delimiter (Parser<MacroInvocLexer> &parser, bool semicolon,
AST::DelimType delimiter, TokenId last_token_id)
{
if (semicolon || delimiter == AST::DelimType::CURLY)
return transcribe_many_stmts (parser, last_token_id);
else
return transcribe_expression (parser);
} // namespace Rust
static std::vector<AST::SingleASTNode>
transcribe_context (MacroExpander::ContextType ctx,
Parser<MacroInvocLexer> &parser, bool semicolon,
AST::DelimType delimiter, TokenId last_token_id)
{
// The flow-chart in order to choose a parsing function is as follows:
//
// [switch special context]
// -- Item --> parser.parse_item();
// -- Trait --> parser.parse_trait_item();
// -- Impl --> parser.parse_impl_item();
// -- Extern --> parser.parse_extern_item();
// -- None --> [has semicolon?]
// -- Yes --> parser.parse_stmt();
// -- No --> [switch invocation.delimiter()]
// -- { } --> parser.parse_stmt();
// -- _ --> parser.parse_expr(); // once!
// If there is a semicolon OR we are expanding a MacroInvocationSemi, then
// we can parse multiple items. Otherwise, parse *one* expression
switch (ctx)
{
case MacroExpander::ContextType::ITEM:
return transcribe_many_items (parser, last_token_id);
break;
case MacroExpander::ContextType::TRAIT:
return transcribe_many_trait_items (parser, last_token_id);
break;
case MacroExpander::ContextType::IMPL:
return transcribe_many_impl_items (parser, last_token_id);
break;
case MacroExpander::ContextType::EXTERN:
return transcribe_many_ext (parser, last_token_id);
break;
default:
return transcribe_on_delimiter (parser, semicolon, delimiter,
last_token_id);
}
}
AST::ASTFragment
MacroExpander::transcribe_rule (
AST::MacroRule &match_rule, AST::DelimTokenTree &invoc_token_tree,
@ -864,18 +960,15 @@ MacroExpander::transcribe_rule (
std::vector<std::unique_ptr<AST::Token>> substituted_tokens
= substitute_context.substitute_tokens ();
// handy for debugging
// for (auto &tok : substituted_tokens)
// rust_debug ("[tok] %s", tok->as_string ().c_str ());
// parse it to an ASTFragment
MacroInvocLexer lex (std::move (substituted_tokens));
Parser<MacroInvocLexer> parser (std::move (lex));
// handy for debugging
// for (auto &tok : substituted_tokens)
// {
// rust_debug ("tok: [%s]", tok->as_string ().c_str ());
// }
auto last_token_id = TokenId::RIGHT_CURLY;
std::vector<AST::SingleASTNode> nodes;
// this is used so we can check that we delimit the stream correctly.
switch (transcribe_tree.get_delim_type ())
@ -905,33 +998,9 @@ MacroExpander::transcribe_rule (
// as a statement (either via ExpressionStatement or
// MacroInvocationWithSemi)
// The flow-chart in order to choose a parsing function is as follows:
//
// [is in item context?]
// -- Yes --> parser.parse_item();
// -- No --> [has semicolon?]
// -- Yes --> parser.parse_stmt();
// -- No --> [switch invocation.delimiter()]
// -- { } --> parser.parse_stmt();
// -- _ --> parser.parse_expr();
// If there is a semicolon OR we are expanding a MacroInvocationSemi, then
// we can parse multiple items. Otherwise, parse *one* expression
if (ctx == ContextType::ITEM)
nodes = transcribe_many_items (parser, last_token_id);
else if (semicolon)
nodes = transcribe_many_stmts (parser, last_token_id);
else
switch (invoc_token_tree.get_delim_type ())
{
case AST::CURLY:
nodes = transcribe_many_stmts (parser, last_token_id);
break;
default:
nodes = transcribe_expression (parser);
break;
}
auto nodes
= transcribe_context (ctx, parser, semicolon,
invoc_token_tree.get_delim_type (), last_token_id);
// emit any errors
if (parser.has_errors ())

3
gcc/rust/expand/rust-macro-expand.h
View File

@ -187,6 +187,9 @@ struct MacroExpander
{
ITEM,
BLOCK,
TRAIT,
IMPL,
EXTERN,
};
ExpansionCfg cfg;

7
gcc/rust/parse/rust-parse-impl.h
View File

@ -5251,12 +5251,11 @@ Parser<ManagedTokenSource>::parse_inherent_impl_item ()
switch (t->get_id ())
{
case IDENTIFIER:
// FIXME: Arthur: Do we need to some lookahead here?
return parse_macro_invocation_semi (outer_attrs);
case SUPER:
case SELF:
case CRATE:
case DOLLAR_SIGN:
// these seem to be SimplePath tokens, so this is a macro invocation semi
return parse_macro_invocation_semi (std::move (outer_attrs));
case PUB: {
// visibility, so not a macro invocation semi - must be constant,
// function, or method
@ -5813,6 +5812,8 @@ Parser<ManagedTokenSource>::parse_external_item ()
const_TokenPtr t = lexer.peek_token ();
switch (t->get_id ())
{
case IDENTIFIER:
return parse_macro_invocation_semi (outer_attrs);
case STATIC_TOK: {
// parse extern static item
lexer.skip_token ();

6
gcc/rust/parse/rust-parse.h
View File

@ -135,6 +135,9 @@ public:
*/
std::unique_ptr<AST::Stmt> parse_stmt (bool allow_no_semi = false);
std::unique_ptr<AST::Type> parse_type ();
std::unique_ptr<AST::ExternalItem> parse_external_item ();
std::unique_ptr<AST::TraitItem> parse_trait_item ();
std::unique_ptr<AST::InherentImplItem> parse_inherent_impl_item ();
AST::PathInExpression parse_path_in_expression ();
std::vector<std::unique_ptr<AST::LifetimeParam> > parse_lifetime_params ();
AST::Visibility parse_visibility ();
@ -285,7 +288,6 @@ private:
AST::AttrVec outer_attrs);
std::unique_ptr<AST::Trait> parse_trait (AST::Visibility vis,
AST::AttrVec outer_attrs);
std::unique_ptr<AST::TraitItem> parse_trait_item ();
std::unique_ptr<AST::TraitItemType>
parse_trait_type (AST::AttrVec outer_attrs);
std::unique_ptr<AST::TraitItemConst>
@ -293,7 +295,6 @@ private:
AST::SelfParam parse_self_param ();
std::unique_ptr<AST::Impl> parse_impl (AST::Visibility vis,
AST::AttrVec outer_attrs);
std::unique_ptr<AST::InherentImplItem> parse_inherent_impl_item ();
std::unique_ptr<AST::InherentImplItem>
parse_inherent_impl_function_or_method (AST::Visibility vis,
AST::AttrVec outer_attrs);
@ -303,7 +304,6 @@ private:
AST::AttrVec outer_attrs);
std::unique_ptr<AST::ExternBlock>
parse_extern_block (AST::Visibility vis, AST::AttrVec outer_attrs);
std::unique_ptr<AST::ExternalItem> parse_external_item ();
AST::NamedFunctionParam parse_named_function_param (AST::AttrVec outer_attrs
= AST::AttrVec ());
AST::Method parse_method ();