Introduce a proper keyword list

The old "keyword" list was used for the lexer, and could therefore not be used with keyword spanning over multiple tokens as those tokens should remain lexed as is. Hence the introduction of a new list macro for keyword exclusive tasks. This also means we can no longer match a token id for each keyword. The token id map has been renamed to keep it's properties. gcc/rust/ChangeLog: * lex/rust-lex.cc (Lexer::classify_keyword): Update keyword map name. * lex/rust-token.h (enum PrimitiveCoreType): Remove some deprecated comments. * util/rust-keyword-values.cc (get_keywords): Update the keyword map name. (RS_TOKEN): Define as empty (RS_TOKEN_KEYWORD_2015): Add the emission value. (RS_TOKEN_KEYWORD_2018): Likewise. * util/rust-keyword-values.h (RS_KEYWORD_LIST): Introduce the keyword list. (RS_TOKEN_KEYWORD_2018): Define multiple new keywords. Signed-off-by: Pierre-Emmanuel Patry <pierre-emmanuel.patry@embecosm.com>
Rust-GCC · Nov 21, 2023 · 34479e5 · 34479e5
1 parent 897532a
commit 34479e5
Show file tree

Hide file tree

Showing 4 changed files with 24 additions and 6 deletions.
diff --git a/gcc/rust/lex/rust-lex.cc b/gcc/rust/lex/rust-lex.cc
@@ -260,7 +260,7 @@ Lexer::replace_current_token (TokenPtr replacement)
 TokenId
 Lexer::classify_keyword (const std::string &str)
 {
-  auto &keywords = Rust::Values::Keywords::keywords;
+  auto &keywords = Rust::Values::Keywords::keywords_tokens;
   auto keyword = keywords.find (str);
 
   if (keyword == keywords.end ())

diff --git a/gcc/rust/lex/rust-token.h b/gcc/rust/lex/rust-token.h
@@ -145,7 +145,6 @@ enum PrimitiveCoreType
   /* Doc Comments */                                                           \
   RS_TOKEN (INNER_DOC_COMMENT, "#![doc]")                                      \
   RS_TOKEN (OUTER_DOC_COMMENT, "#[doc]")                                       \
-  /* have "weak" union and 'static keywords? */                                \
   RS_TOKEN_KEYWORD_2015 (ABSTRACT, "abstract") /* unused */                    \
   RS_TOKEN_KEYWORD_2015 (AS, "as")                                             \
   RS_TOKEN_KEYWORD_2018 (ASYNC, "async") /* unused */                          \
@@ -157,7 +156,6 @@ enum PrimitiveCoreType
   RS_TOKEN_KEYWORD_2015 (CONST, "const")                                       \
   RS_TOKEN_KEYWORD_2015 (CONTINUE, "continue")                                 \
   RS_TOKEN_KEYWORD_2015 (CRATE, "crate")                                       \
-  /* FIXME: Do we need to add $crate (DOLLAR_CRATE) as a reserved kw? */       \
   RS_TOKEN_KEYWORD_2015 (DO, "do") /* unused */                                \
   RS_TOKEN_KEYWORD_2018 (DYN, "dyn")                                           \
   RS_TOKEN_KEYWORD_2015 (ELSE, "else")                                         \

diff --git a/gcc/rust/util/rust-keyword-values.cc b/gcc/rust/util/rust-keyword-values.cc
@@ -38,7 +38,18 @@ get_keywords ()
   return m;
 }
 
-const std::map<std::string, TokenId> Keywords::keywords = get_keywords ();
+const std::map<std::string, TokenId> Keywords::keywords_tokens
+  = get_keywords ();
+
+const std::set<std::string> Keywords::keywords = {
+#define RS_TOKEN(x, y)
+#define RS_TOKEN_KEYWORD_2015(tok, key) {key},
+#define RS_TOKEN_KEYWORD_2018 RS_TOKEN_KEYWORD_2015
+  RS_KEYWORD_LIST
+#undef RS_TOKEN_KEYWORD_2015
+#undef RS_TOKEN_KEYWORD_2018
+#undef RS_TOKEN
+};
 
 } // namespace Values
 } // namespace Rust
diff --git a/gcc/rust/util/rust-keyword-values.h b/gcc/rust/util/rust-keyword-values.h
@@ -21,21 +21,30 @@
 
 #include "rust-token.h"
 
+// Append keywords made from multiple tokens to the existing token-keyword list
+#define RS_KEYWORD_LIST                                                        \
+  RS_TOKEN_LIST                                                                \
+  RS_TOKEN_KEYWORD_2015 (DOLLAR_CRATE, "$crate")                               \
+  RS_TOKEN_KEYWORD_2015 (PATH_ROOT, "{{root}}")                                \
+  RS_TOKEN_KEYWORD_2015 (STATIC_LIFETIME, "'static")                           \
+  RS_TOKEN_KEYWORD_2015 (UNDERSCORE_LIFETIME, "'_")
+
 namespace Rust {
 namespace Values {
 
 // TODO: Change this to a namespace + inline constexpr in the future
 class Keywords
 {
 public:
-  const static std::map<std::string, TokenId> keywords;
+  const static std::map<std::string, TokenId> keywords_tokens;
 
+  const static std::set<std::string> keywords;
   // Rust keyword values
 public:
 #define RS_TOKEN(x, y)
 #define RS_TOKEN_KEYWORD_2015(tok, key) static constexpr auto &tok = key;
 #define RS_TOKEN_KEYWORD_2018 RS_TOKEN_KEYWORD_2015
-  RS_TOKEN_LIST
+  RS_KEYWORD_LIST
 #undef RS_TOKEN_KEYWORD_2015
 #undef RS_TOKEN_KEYWORD_2018
 #undef RS_TOKEN