From 3a6bb1fd925bbcc2aa94a89482b7c9340b82e1b1 Mon Sep 17 00:00:00 2001
From: Code Ass <aycabta@gmail.com>
Date: Fri, 11 Aug 2017 19:37:21 +0900
Subject: [PATCH 1/2] Fix creation of symbol token

Symbol tokens are created form TkIDENTIFIER with ":",
but there are several other cases.
Subclasses of TkId can become symbol token with ":",
for example reserved keywords can become symbol, "{ undef: undef }".
This commit fixes it.
---
 lib/rdoc/ruby_lex.rb | 30 +++++++++++++++---------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/lib/rdoc/ruby_lex.rb b/lib/rdoc/ruby_lex.rb
index 7abcc37777..0a8f181d1b 100644
--- a/lib/rdoc/ruby_lex.rb
+++ b/lib/rdoc/ruby_lex.rb
@@ -1046,12 +1046,7 @@ def identify_identifier
                   @indent_stack.push token_c
                 end
               else
-                if peek(0) == ':' and !peek_match?(/^::/)
-                  token.concat getc
-                  token_c = TkSYMBOL
-                else
-                  token_c = TkIDENTIFIER
-                end
+                token_c = TkIDENTIFIER
               end
 
             elsif DEINDENT_CLAUSE.include?(token)
@@ -1063,6 +1058,10 @@ def identify_identifier
             @lex_state = :EXPR_END
           end
         end
+        if token_c.ancestors.include?(TkId) and peek(0) == ':' and !peek_match?(/^::/)
+          token.concat getc
+          token_c = TkSYMBOL
+        end
         return Token(token_c, token)
       end
     end
@@ -1081,19 +1080,20 @@ def identify_identifier
 
     if token[0, 1] =~ /[A-Z]/
       if token[-1] =~ /[!?]/
-        return Token(TkIDENTIFIER, token)
+        token_c = TkIDENTIFIER
       else
-        return Token(TkCONSTANT, token)
+        token_c = TkCONSTANT
       end
     elsif token[token.size - 1, 1] =~ /[!?]/
-      return Token(TkFID, token)
+      token_c = TkFID
     else
-      if peek(0) == ':' and !peek_match?(/^::/)
-        token.concat getc
-        return Token(TkSYMBOL, token)
-      else
-        return Token(TkIDENTIFIER, token)
-      end
+      token_c = TkIDENTIFIER
+    end
+    if peek(0) == ':' and !peek_match?(/^::/)
+      token.concat getc
+      return Token(TkSYMBOL, token)
+    else
+      return Token(token_c, token)
     end
   end
 

From cb4787fad18eb7976b2988d96040cff5374490fe Mon Sep 17 00:00:00 2001
From: Code Ass <aycabta@gmail.com>
Date: Fri, 11 Aug 2017 19:56:57 +0900
Subject: [PATCH 2/2] Add test_class_tokenize_particular_kind_of_symbols

---
 test/test_rdoc_ruby_lex.rb | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/test/test_rdoc_ruby_lex.rb b/test/test_rdoc_ruby_lex.rb
index d8a920f42c..279cdf99ec 100644
--- a/test/test_rdoc_ruby_lex.rb
+++ b/test/test_rdoc_ruby_lex.rb
@@ -520,6 +520,33 @@ def test_class_tokenize_symbol
     assert_equal expected, tokens
   end
 
+  def test_class_tokenize_particular_kind_of_symbols
+    tokens = RDoc::RubyLex.tokenize '{ Thomas: :Thomas, Dave!: :Dave!, undef: :undef }', nil
+
+    expected = [
+      @TK::TkLBRACE.new( 0, 1,  0, "{"),
+      @TK::TkSPACE .new( 1, 1,  1, " "),
+      @TK::TkSYMBOL.new( 2, 1,  2, "Thomas:"),
+      @TK::TkSPACE .new( 9, 1,  9, " "),
+      @TK::TkSYMBOL.new(10, 1, 10, ":Thomas"),
+      @TK::TkCOMMA .new(17, 1, 17, ","),
+      @TK::TkSPACE .new(18, 1, 18, " "),
+      @TK::TkSYMBOL.new(19, 1, 19, "Dave!:"),
+      @TK::TkSPACE .new(25, 1, 25, " "),
+      @TK::TkSYMBOL.new(26, 1, 26, ":Dave!"),
+      @TK::TkCOMMA .new(32, 1, 32, ","),
+      @TK::TkSPACE .new(33, 1, 33, " "),
+      @TK::TkSYMBOL.new(34, 1, 34, "undef:"),
+      @TK::TkSPACE .new(40, 1, 40, " "),
+      @TK::TkSYMBOL.new(41, 1, 41, ":undef"),
+      @TK::TkSPACE .new(47, 1, 47, " "),
+      @TK::TkRBRACE.new(48, 1, 48, "}"),
+      @TK::TkNL    .new(49, 1, 49, "\n"),
+    ]
+
+    assert_equal expected, tokens
+  end
+
   def test_unary_minus
     ruby_lex = RDoc::RubyLex.new("-1", nil)
     assert_equal("-1", ruby_lex.token.value)