Merge #1051

1051: macros: Add remaining restrictions for follow-set restrictions r=CohenArthur a=CohenArthur

Adds the remaining restrictions for follow-set ambiguities in macros.
This means adding the remaining allowed tokens for all fragment
specifiers with follow-up restrictions, as well as handling allowed
fragment specifiers in certain cases. For example, :vis specifiers can
sometimes be followed by fragments, if they have the :ident, :ty or
:path specifier. Likewise for :path and :ty which can be followed by a
:block.

Finally, we also allow *any* fragment after a matcher: Since the matcher
is delimiter by parentheses, brackets or curlies, anything is allowed
afterwards.

Some edge cases or allowed tokens that we cannot handle yet remain, for which FIXMEs exist. I'll open up corresponding issues. 

Addresses #947 

Co-authored-by: Arthur Cohen <arthur.cohen@embecosm.com>

gcc/rust/ast/rust-macro.h

@@ -128,19 +128,29 @@ public:
       }
   }
 
-  bool has_follow_set_restrictions ()
+  bool has_follow_set_restrictions () const
   {
     switch (kind)
       {
       case EXPR:
       case STMT:
-	// FIXME: Add the following cases once we can handle them properly
-	// in `is_match_compatible()`
-	// case PAT:
-	// // case PAT_PARAM: FIXME: Doesn't <metavar>:pat_param exist?
-	// case PATH:
-	// case TY:
-	// case VIS:
+      case PAT:
+      case PATH:
+      case TY:
+      case VIS:
+	return true;
+      default:
+	return false;
+      }
+  }
+
+  bool has_follow_set_fragment_restrictions () const
+  {
+    switch (kind)
+      {
+      case PAT:
+      case TY:
+      case VIS:
 	return true;
       default:
 	return false;
@@ -188,7 +198,7 @@ public:
   }
 
   Identifier get_ident () const { return ident; }
-  MacroFragSpec get_frag_spec () const { return frag_spec; }
+  const MacroFragSpec &get_frag_spec () const { return frag_spec; }
 
 protected:
   /* Use covariance to implement clone function as returning this object rather

gcc/rust/lex/rust-token.h

@@ -165,6 +165,7 @@ enum PrimitiveCoreType
   RS_TOKEN_KEYWORD (CONST, "const")                                            \
   RS_TOKEN_KEYWORD (CONTINUE, "continue")                                      \
   RS_TOKEN_KEYWORD (CRATE, "crate")                                            \
+  /* FIXME: Do we need to add $crate (DOLLAR_CRATE) as a reserved kw? */       \
   RS_TOKEN_KEYWORD (DO, "do") /* unused */                                     \
   RS_TOKEN_KEYWORD (DYN, "dyn")                                                \
   RS_TOKEN_KEYWORD (ELSE, "else")                                              \

gcc/rust/parse/rust-parse.cc

@@ -93,15 +93,83 @@ extract_module_path (const AST::AttrVec &inner_attrs,
   return path;
 }
 
+template <typename T>
+static bool
+contains (std::vector<T> &vec, T elm)
+{
+  return std::find (vec.begin (), vec.end (), elm) != vec.end ();
+}
+
+static bool
+peculiar_fragment_match_compatible_fragment (
+  const AST::MacroFragSpec &last_spec, const AST::MacroFragSpec &spec,
+  Location match_locus)
+{
+  static std::unordered_map<AST::MacroFragSpec::Kind,
+			    std::vector<AST::MacroFragSpec::Kind>>
+    fragment_follow_set
+    = {{AST::MacroFragSpec::PATH, {AST::MacroFragSpec::BLOCK}},
+       {AST::MacroFragSpec::TY, {AST::MacroFragSpec::BLOCK}},
+       {AST::MacroFragSpec::VIS,
+	{AST::MacroFragSpec::IDENT, AST::MacroFragSpec::TY,
+	 AST::MacroFragSpec::PATH}}};
+
+  auto is_valid
+    = contains (fragment_follow_set[last_spec.get_kind ()], spec.get_kind ());
+
+  if (!is_valid)
+    rust_error_at (
+      match_locus,
+      "fragment specifier %<%s%> is not allowed after %<%s%> fragments",
+      spec.as_string ().c_str (), last_spec.as_string ().c_str ());
+
+  return is_valid;
+}
+
 static bool
 peculiar_fragment_match_compatible (AST::MacroMatchFragment &last_match,
 				    AST::MacroMatch &match)
 {
   static std::unordered_map<AST::MacroFragSpec::Kind, std::vector<TokenId>>
-    follow_set = {
-      {AST::MacroFragSpec::EXPR, {MATCH_ARROW, COMMA, SEMICOLON}},
-      {AST::MacroFragSpec::STMT, {MATCH_ARROW, COMMA, SEMICOLON}},
-    };
+    follow_set
+    = {{AST::MacroFragSpec::EXPR, {MATCH_ARROW, COMMA, SEMICOLON}},
+       {AST::MacroFragSpec::STMT, {MATCH_ARROW, COMMA, SEMICOLON}},
+       {AST::MacroFragSpec::PAT, {MATCH_ARROW, COMMA, EQUAL, PIPE, IF, IN}},
+       {AST::MacroFragSpec::PATH,
+	{MATCH_ARROW, COMMA, EQUAL, PIPE, SEMICOLON, COLON, RIGHT_ANGLE,
+	 RIGHT_SHIFT, LEFT_SQUARE, LEFT_CURLY, AS, WHERE}},
+       {AST::MacroFragSpec::TY,
+	{MATCH_ARROW, COMMA, EQUAL, PIPE, SEMICOLON, COLON, RIGHT_ANGLE,
+	 RIGHT_SHIFT, LEFT_SQUARE, LEFT_CURLY, AS, WHERE}},
+       {AST::MacroFragSpec::VIS,
+	{
+	  COMMA,
+	  IDENTIFIER /* FIXME: Other than `priv` */,
+	  LEFT_PAREN,
+	  LEFT_SQUARE,
+	  EXCLAM,
+	  ASTERISK,
+	  AMP,
+	  LOGICAL_AND,
+	  QUESTION_MARK,
+	  LIFETIME,
+	  LEFT_ANGLE,
+	  LEFT_SHIFT,
+	  SUPER,
+	  SELF,
+	  SELF_ALIAS,
+	  EXTERN_TOK,
+	  CRATE,
+	  UNDERSCORE,
+	  FOR,
+	  IMPL,
+	  FN_TOK,
+	  UNSAFE,
+	  TYPEOF,
+	  DYN
+	  // FIXME: Add Non kw identifiers
+	  // FIXME: Add $crate as valid
+	}}};
 
   Location error_locus = match.get_match_locus ();
 
@@ -117,9 +185,7 @@ peculiar_fragment_match_compatible (AST::MacroMatchFragment &last_match,
 	auto tok = static_cast<AST::Token *> (&match);
 	auto &allowed_toks
 	  = follow_set[last_match.get_frag_spec ().get_kind ()];
-	auto is_valid = std::find (allowed_toks.begin (), allowed_toks.end (),
-				   tok->get_id ())
-			!= allowed_toks.end ();
+	auto is_valid = contains (allowed_toks, tok->get_id ());
 	if (!is_valid)
 	  // FIXME: Add hint about allowed fragments
 	  rust_error_at (tok->get_match_locus (),
@@ -143,10 +209,17 @@ peculiar_fragment_match_compatible (AST::MacroMatchFragment &last_match,
 	  error_locus = matches.front ()->get_match_locus ();
 	break;
       }
-    default:
+      case AST::MacroMatch::Fragment: {
+	auto last_spec = last_match.get_frag_spec ();
+	auto fragment = static_cast<AST::MacroMatchFragment *> (&match);
+	if (last_spec.has_follow_set_fragment_restrictions ())
+	  return peculiar_fragment_match_compatible_fragment (
+	    last_spec, fragment->get_frag_spec (), match.get_match_locus ());
+      }
       break;
     }
 
+  // FIXME: Improve error message
   rust_error_at (error_locus, "fragment not allowed after %<%s%> fragment",
 		 last_match.get_frag_spec ().as_string ().c_str ());
 
@@ -213,16 +286,7 @@ is_match_compatible (AST::MacroMatch &last_match, AST::MacroMatch &match)
 	  return true;
 	break;
       }
-      case AST::MacroMatch::Matcher: {
-	// Likewise for another matcher
-	auto matcher = static_cast<AST::MacroMatcher *> (&last_match);
-	new_last = get_back_ptr (matcher->get_matches ());
-	// If there are no matches in the matcher, then it can be followed by
-	// anything
-	if (!new_last)
-	  return true;
-	break;
-      }
+    case AST::MacroMatch::Matcher:
     case AST::MacroMatch::Tok:
       return true;
     }

gcc/testsuite/rust/compile/macro33.rs

@@ -0,0 +1,5 @@
+macro_rules! forbidden_frag {
+    ($t:ty $not_block:ident) => {{}}; // { dg-error "fragment specifier .ident. is not allowed after .ty. fragments" }
+                                      // { dg-error "required first macro rule in macro rules definition could not be parsed" "" { target *-*-* } .-1 }
+                                      // { dg-error "failed to parse item in crate" "" { target *-*-* } .-2 }
+}

gcc/testsuite/rust/compile/macro34.rs

@@ -0,0 +1,3 @@
+macro_rules! allowed_after_expr_matcher {
+    (($t:expr) bok) => {{}}; // follow-set restrictions do not apply after a matcher, but they do apply inside the matcher
+}

gcc/testsuite/rust/compile/macro35.rs

@@ -0,0 +1,7 @@
+macro_rules! inside_matcher {
+    (($e:expr tok) tok) => {{}}; // { dg-error "token .tok. is not allowed after .expr. fragment" }
+                                 // { dg-error "failed to parse macro matcher" "" { target *-*-* } .-1 }
+                                 // { dg-error "failed to parse macro match" "" { target *-*-* } .-2 }
+                                 // { dg-error "required first macro rule" "" { target *-*-* } .-3 }
+                                 // { dg-error "failed to parse item in crate" "" { target *-*-* } .-4 }
+}

gcc/testsuite/rust/compile/macro36.rs

@@ -0,0 +1,3 @@
+macro_rules! ty_allowed {
+    ($t:ty $b:block) => {{}};
+}