From 434f9c661b0e72a7de18ffc001b3b31cdf65bda9 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Mon, 24 Jul 2017 00:05:46 +0900 Subject: [PATCH 1/5] Handle JSON-style hash key as symbol --- lib/rdoc/ruby_lex.rb | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/lib/rdoc/ruby_lex.rb b/lib/rdoc/ruby_lex.rb index 44cca09f73..4460120698 100644 --- a/lib/rdoc/ruby_lex.rb +++ b/lib/rdoc/ruby_lex.rb @@ -948,7 +948,12 @@ def identify_identifier @indent_stack.push token_c end else - token_c = TkIDENTIFIER + if peek(0) == ':' + token.concat getc + token_c = TkSYMBOL + else + token_c = TkIDENTIFIER + end end elsif DEINDENT_CLAUSE.include?(token) @@ -981,7 +986,12 @@ def identify_identifier elsif token[token.size - 1, 1] =~ /[!?]/ return Token(TkFID, token) else - return Token(TkIDENTIFIER, token) + if peek(0) == ':' + token.concat getc + return Token(TkSYMBOL, token) + else + return Token(TkIDENTIFIER, token) + end end end From 3a286401f9a885761ddfbd0dad3a36ab7ef0f9ae Mon Sep 17 00:00:00 2001 From: Code Ass Date: Mon, 24 Jul 2017 00:08:32 +0900 Subject: [PATCH 2/5] Fix JSON-style hash key test --- test/test_rdoc_ruby_lex.rb | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test/test_rdoc_ruby_lex.rb b/test/test_rdoc_ruby_lex.rb index a57bfe3a80..f9f7a1882a 100644 --- a/test/test_rdoc_ruby_lex.rb +++ b/test/test_rdoc_ruby_lex.rb @@ -78,13 +78,13 @@ def test_class_tokenize_hash_symbol tokens = RDoc::RubyLex.tokenize '{ class:"foo" }', nil expected = [ - @TK::TkLBRACE .new( 0, 1, 0, '{'), - @TK::TkSPACE .new( 1, 1, 1, ' '), - @TK::TkIDENTIFIER.new( 2, 1, 2, 'class'), - @TK::TkSYMBOL .new( 7, 1, 7, ':"foo"'), - @TK::TkSPACE .new(13, 1, 13, ' '), - @TK::TkRBRACE .new(14, 1, 14, '}'), - @TK::TkNL .new(15, 1, 15, "\n"), + @TK::TkLBRACE.new( 0, 1, 0, '{'), + @TK::TkSPACE .new( 1, 1, 1, ' '), + @TK::TkSYMBOL.new( 2, 1, 2, 'class:'), + @TK::TkSTRING.new( 8, 1, 8, '"foo"'), + @TK::TkSPACE .new(13, 1, 13, ' '), + @TK::TkRBRACE.new(14, 1, 14, '}'), + @TK::TkNL .new(15, 1, 15, "\n"), ] assert_equal expected, tokens From dec804f40cf195c27a4827c454eb39bcd93846b5 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Mon, 24 Jul 2017 01:15:55 +0900 Subject: [PATCH 3/5] Fix test what has a keyword argument --- test/test_rdoc_ruby_lex.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/test_rdoc_ruby_lex.rb b/test/test_rdoc_ruby_lex.rb index f9f7a1882a..076b74e8bc 100644 --- a/test/test_rdoc_ruby_lex.rb +++ b/test/test_rdoc_ruby_lex.rb @@ -389,8 +389,7 @@ def test_class_tokenize_symbol expected = [ @TK::TkIDENTIFIER.new( 0, 1, 0, 'scope'), @TK::TkSPACE .new( 5, 1, 5, ' '), - @TK::TkIDENTIFIER.new( 6, 1, 6, 'module'), - @TK::TkCOLON .new(12, 1, 12, ':'), + @TK::TkSYMBOL .new( 6, 1, 6, 'module:'), @TK::TkSPACE .new(13, 1, 13, ' '), @TK::TkSYMBOL .new(14, 1, 14, ':v1'), @TK::TkNL .new(17, 1, 17, "\n"), From f6af91b0bed78dfbee16295e5ba64c5cdab0cf6c Mon Sep 17 00:00:00 2001 From: Code Ass Date: Mon, 24 Jul 2017 05:20:22 +0900 Subject: [PATCH 4/5] Remove :: from hash key detection condition --- lib/rdoc/ruby_lex.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/rdoc/ruby_lex.rb b/lib/rdoc/ruby_lex.rb index 4460120698..2c5a926b66 100644 --- a/lib/rdoc/ruby_lex.rb +++ b/lib/rdoc/ruby_lex.rb @@ -948,7 +948,7 @@ def identify_identifier @indent_stack.push token_c end else - if peek(0) == ':' + if peek(0) == ':' and !peek_match?(/^::/) token.concat getc token_c = TkSYMBOL else @@ -986,7 +986,7 @@ def identify_identifier elsif token[token.size - 1, 1] =~ /[!?]/ return Token(TkFID, token) else - if peek(0) == ':' + if peek(0) == ':' and !peek_match?(/^::/) token.concat getc return Token(TkSYMBOL, token) else From f291a7de4c620cf6b51fdff3fa7f5c7a617cb0ee Mon Sep 17 00:00:00 2001 From: Code Ass Date: Mon, 24 Jul 2017 05:29:37 +0900 Subject: [PATCH 5/5] Add test_class_tokenize_double_colon_is_not_hash_symbol --- test/test_rdoc_ruby_lex.rb | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/test/test_rdoc_ruby_lex.rb b/test/test_rdoc_ruby_lex.rb index 076b74e8bc..7e446c5b8e 100644 --- a/test/test_rdoc_ruby_lex.rb +++ b/test/test_rdoc_ruby_lex.rb @@ -90,6 +90,21 @@ def test_class_tokenize_hash_symbol assert_equal expected, tokens end + def test_class_tokenize_double_colon_is_not_hash_symbol + tokens = RDoc::RubyLex.tokenize 'self.class::Row', nil + + expected = [ + @TK::TkSELF .new( 0, 1, 0, "self"), + @TK::TkDOT .new( 4, 1, 4, "."), + @TK::TkIDENTIFIER.new( 5, 1, 5, "class"), + @TK::TkCOLON2 .new(10, 1, 10, "::"), + @TK::TkCONSTANT .new(12, 1, 12, "Row"), + @TK::TkNL .new(15, 1, 15, "\n"), + ] + + assert_equal expected, tokens + end + def test_class_tokenize_heredoc_CR_NL tokens = RDoc::RubyLex.tokenize <<-RUBY, nil string = <<-STRING\r