Merge #938

938: First pass at declarative macro expansion  r=philberty a=philberty

This does not support repetition matchers but it supports simple
declarative macros and transcribes them. The approach taken here is that
we reuse our existing parser to call the apropriate functions as specified
as part of the MacroFragmentType enum if the parser does not have errors
parsing that item then it must be a match.
    
Then once we match a rule we have a map of the token begin/end offsets
for each fragment match, this is then used to adjust and create a new token
stream for the macro rule definition so that when we feed it to the parser
the tokens are already substituted. The resulting expression or item is
then attached to the respective macro invocation and this is then name
resolved and used for hir lowering.
    
Fixes gcc-mirror#17 gcc-mirror#22
Addresses #573

Co-authored-by: Philip Herron <[email protected]>

Author: bors[bot]

gcc/rust/ast/rust-ast-full-test.cc

@@ -4387,13 +4387,6 @@ std::vector<std::unique_ptr<Token> >
 DelimTokenTree::to_token_stream () const
 {
   std::vector<std::unique_ptr<Token> > tokens;
-
-  // simulate presence of delimiters
-  const_TokenPtr left_paren
-    = Rust::Token::make (LEFT_PAREN, Linemap::unknown_location ());
-  tokens.push_back (
-    std::unique_ptr<Token> (new Token (std::move (left_paren))));
-
   for (const auto &tree : token_trees)
     {
       std::vector<std::unique_ptr<Token> > stream = tree->to_token_stream ();
@@ -4402,13 +4395,7 @@ DelimTokenTree::to_token_stream () const
 		     std::make_move_iterator (stream.end ()));
     }
 
-  const_TokenPtr right_paren
-    = Rust::Token::make (RIGHT_PAREN, Linemap::unknown_location ());
-  tokens.push_back (
-    std::unique_ptr<Token> (new Token (std::move (right_paren))));
-
   tokens.shrink_to_fit ();
-
   return tokens;
 }
 

gcc/rust/ast/rust-ast.h

@@ -22,17 +22,13 @@
 
 #include "rust-system.h"
 #include "rust-hir-map.h"
-
-// gccrs imports
-// required for AST::Token
 #include "rust-token.h"
 #include "rust-location.h"
 
 namespace Rust {
 // TODO: remove typedefs and make actual types for these
 typedef std::string Identifier;
 typedef int TupleIndex;
-
 struct Session;
 
 namespace AST {
@@ -48,34 +44,6 @@ enum DelimType
   CURLY
 };
 
-// Base AST node object - TODO is this really required or useful? Where to draw
-// line?
-/*class Node {
-  public:
-    // Gets node's Location.
-    Location get_locus() const {
-	return loc;
-    }
-
-    // Sets node's Location.
-    void set_locus(Location loc_) {
-	loc = loc_;
-    }
-
-    // Get node output as a string. Pure virtual.
-    virtual std::string as_string() const = 0;
-
-    virtual ~Node() {}
-
-    // TODO: constructor including Location? Make all derived classes have
-Location?
-
-  private:
-    // The node's location.
-    Location loc;
-};*/
-// decided to not have node as a "node" would never need to be stored
-
 // forward decl for use in token tree method
 class Token;
 
@@ -108,6 +76,14 @@ class TokenTree
 class MacroMatch
 {
 public:
+  enum MacroMatchType
+  {
+    Fragment,
+    Repetition,
+    Matcher,
+    Tok
+  };
+
   virtual ~MacroMatch () {}
 
   virtual std::string as_string () const = 0;
@@ -121,6 +97,8 @@ class MacroMatch
 
   virtual void accept_vis (ASTVisitor &vis) = 0;
 
+  virtual MacroMatchType get_macro_match_type () const = 0;
+
 protected:
   // pure virtual clone implementation
   virtual MacroMatch *clone_macro_match_impl () const = 0;
@@ -234,6 +212,11 @@ class Token : public TokenTree, public MacroMatch
   // Get a new token pointer copy.
   const_TokenPtr get_tok_ptr () const { return tok_ref; }
 
+  MacroMatchType get_macro_match_type () const override
+  {
+    return MacroMatchType::Tok;
+  }
+
 protected:
   // No virtual for now as not polymorphic but can be in future
   /*virtual*/ Token *clone_token_impl () const { return new Token (*this); }
@@ -788,6 +771,13 @@ class DelimTokenTree : public TokenTree, public AttrInput
   {
     return AttrInput::AttrInputType::TOKEN_TREE;
   }
+
+  std::vector<std::unique_ptr<TokenTree> > &get_token_trees ()
+  {
+    return token_trees;
+  }
+
+  DelimType get_delim_type () const { return delim_type; }
 };
 
 /* Forward decl - definition moved to rust-expr.h as it requires LiteralExpr to
@@ -1485,6 +1475,160 @@ struct MacroInvocData
   }
 };
 
+class SingleASTNode
+{
+public:
+  enum NodeType
+  {
+    EXPRESSION,
+    ITEM,
+    STMT,
+  };
+
+  SingleASTNode (std::unique_ptr<Expr> expr)
+    : type (EXPRESSION), expr (std::move (expr)), item (nullptr), stmt (nullptr)
+  {}
+
+  SingleASTNode (std::unique_ptr<Item> item)
+    : type (ITEM), expr (nullptr), item (std::move (item)), stmt (nullptr)
+  {}
+
+  SingleASTNode (std::unique_ptr<Stmt> stmt)
+    : type (STMT), expr (nullptr), item (nullptr), stmt (std::move (stmt))
+  {}
+
+  SingleASTNode (SingleASTNode const &other)
+  {
+    type = other.type;
+    switch (type)
+      {
+      case EXPRESSION:
+	expr = other.expr->clone_expr ();
+	break;
+
+      case ITEM:
+	item = other.item->clone_item ();
+	break;
+
+      case STMT:
+	stmt = other.stmt->clone_stmt ();
+	break;
+      }
+  }
+
+  SingleASTNode operator= (SingleASTNode const &other)
+  {
+    type = other.type;
+    switch (type)
+      {
+      case EXPRESSION:
+	expr = other.expr->clone_expr ();
+	break;
+
+      case ITEM:
+	item = other.item->clone_item ();
+	break;
+
+      case STMT:
+	stmt = other.stmt->clone_stmt ();
+	break;
+      }
+    return *this;
+  }
+
+  SingleASTNode (SingleASTNode &&other) = default;
+  SingleASTNode &operator= (SingleASTNode &&other) = default;
+
+  std::unique_ptr<Expr> &get_expr ()
+  {
+    rust_assert (type == EXPRESSION);
+    return expr;
+  }
+
+  std::unique_ptr<Item> &get_item ()
+  {
+    rust_assert (type == ITEM);
+    return item;
+  }
+
+  std::unique_ptr<Stmt> &get_stmt ()
+  {
+    rust_assert (type == STMT);
+    return stmt;
+  }
+
+  void accept_vis (ASTVisitor &vis)
+  {
+    switch (type)
+      {
+      case EXPRESSION:
+	expr->accept_vis (vis);
+	break;
+
+      case ITEM:
+	item->accept_vis (vis);
+	break;
+
+      case STMT:
+	stmt->accept_vis (vis);
+	break;
+      }
+  }
+
+private:
+  NodeType type;
+
+  // FIXME make this a union
+  std::unique_ptr<Expr> expr;
+  std::unique_ptr<Item> item;
+  std::unique_ptr<Stmt> stmt;
+};
+
+/* Basically, a "fragment" that can be incorporated into the AST, created as
+ * a result of macro expansion. Really annoying to work with due to the fact
+ * that macros can really expand to anything. As such, horrible representation
+ * at the moment. */
+class ASTFragment
+{
+private:
+  /* basic idea: essentially, a vector of tagged unions of different AST node
+   * types. Now, this could actually be stored without a tagged union if the
+   * different AST node types had a unified parent, but that would create
+   * issues with the diamond problem or significant performance penalties. So
+   * a tagged union had to be used instead. A vector is used to represent the
+   * ability for a macro to expand to two statements, for instance. */
+
+  std::vector<SingleASTNode> nodes;
+
+public:
+  ASTFragment (std::vector<SingleASTNode> nodes) : nodes (std::move (nodes)) {}
+
+  ASTFragment (ASTFragment const &other)
+  {
+    nodes.clear ();
+    nodes.reserve (other.nodes.size ());
+    for (auto &n : other.nodes)
+      {
+	nodes.push_back (n);
+      }
+  }
+
+  ASTFragment &operator= (ASTFragment const &other)
+  {
+    nodes.clear ();
+    nodes.reserve (other.nodes.size ());
+    for (auto &n : other.nodes)
+      {
+	nodes.push_back (n);
+      }
+    return *this;
+  }
+
+  static ASTFragment create_empty () { return ASTFragment ({}); }
+
+  std::vector<SingleASTNode> &get_nodes () { return nodes; }
+};
+
 /* A macro invocation item (or statement) AST node (i.e. semi-coloned macro
  * invocation) */
 class MacroInvocationSemi : public MacroItem,
@@ -1496,14 +1640,20 @@ class MacroInvocationSemi : public MacroItem,
   std::vector<Attribute> outer_attrs;
   MacroInvocData invoc_data;
   Location locus;
+  NodeId node_id;
+
+  // this is the expanded macro
+  ASTFragment fragment;
 
 public:
   std::string as_string () const override;
 
   MacroInvocationSemi (MacroInvocData invoc_data,
 		       std::vector<Attribute> outer_attrs, Location locus)
     : outer_attrs (std::move (outer_attrs)),
-      invoc_data (std::move (invoc_data)), locus (locus)
+      invoc_data (std::move (invoc_data)), locus (locus),
+      node_id (Analysis::Mappings::get ()->get_next_node_id ()),
+      fragment (ASTFragment::create_empty ())
   {}
 
   void accept_vis (ASTVisitor &vis) override;
@@ -1527,6 +1677,14 @@ class MacroInvocationSemi : public MacroItem,
 
   Location get_locus () const override final { return locus; }
 
+  MacroInvocData &get_invoc_data () { return invoc_data; }
+
+  ASTFragment &get_fragment () { return fragment; }
+
+  void set_fragment (ASTFragment &&f) { fragment = std::move (f); }
+
+  NodeId get_macro_node_id () const { return node_id; }
+
 protected:
   MacroInvocationSemi *clone_macro_invocation_semi_impl () const
   {