From 8f7742f682eb2fe5ee0110bb58f7a977d8ef2de1 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Wed, 26 Jul 2017 14:59:58 +0900 Subject: [PATCH 1/2] Handle TkPLUS or TkMINUS and number as TkINTEGER For example, lexical analyser splitted sign character and follow number such as "-5r" to "-" and "5r", "+1.08" to "+" and "1.08". But this "-" or "+" is a part of number and not operator. So this Pull Request joins it. --- lib/rdoc/ruby_lex.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/rdoc/ruby_lex.rb b/lib/rdoc/ruby_lex.rb index 7488b80773..bab8c6e7e0 100644 --- a/lib/rdoc/ruby_lex.rb +++ b/lib/rdoc/ruby_lex.rb @@ -376,6 +376,10 @@ def token else tk = tk1 end + elsif (TkPLUS === tk or TkMINUS === tk) and peek(0) =~ /\d/ then + tk1 = token + set_token_position tk.seek, tk.line_no, tk.char_no + tk = Token(tk1.class, tk.text + tk1.text) end # Tracer.off tk From fc22e97fbc666b2e6a4bcf4193b6b5eeaa5ef8d5 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Tue, 22 Aug 2017 17:11:38 +0900 Subject: [PATCH 2/2] Add test_class_tokenize_number_with_sign_character --- test/test_rdoc_ruby_lex.rb | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/test/test_rdoc_ruby_lex.rb b/test/test_rdoc_ruby_lex.rb index 5ded86fd8c..0bcaa45b17 100644 --- a/test/test_rdoc_ruby_lex.rb +++ b/test/test_rdoc_ruby_lex.rb @@ -339,6 +339,19 @@ def test_class_tokenize_regexp_escape assert_equal expected, tokens end + def test_class_tokenize_number_with_sign_character + tokens = RDoc::RubyLex.tokenize "+3--3r", nil + + expected = [ + @TK::TkINTEGER .new(0, 1, 0, "+3"), + @TK::TkMINUS .new(2, 1, 2, "-"), + @TK::TkRATIONAL.new(3, 1, 3, "-3r"), + @TK::TkNL .new(6, 1, 6, "\n"), + ] + + assert_equal expected, tokens + end + def test_class_tokenize_string tokens = RDoc::RubyLex.tokenize "'hi'", nil