dep: change java html dep to neko-htmlunit

flavorjones · flavorjones · commit 277db2eadc23 · 2022-05-10T09:48:07.000-04:00
and update the implementation and the tests
diff --git a/ext/java/nokogiri/Html4ElementDescription.java b/ext/java/nokogiri/Html4ElementDescription.java
@@ -6,7 +6,7 @@
 import java.util.List;
 import java.util.Map;
 
-import org.cyberneko.html.HTMLElements;
+import net.sourceforge.htmlunit.cyberneko.HTMLElements;
 import org.jruby.Ruby;
 import org.jruby.RubyClass;
 import org.jruby.RubyObject;
@@ -24,6 +24,7 @@
 public class Html4ElementDescription extends RubyObject
 {
   private static final long serialVersionUID = 1L;
+  private static final HTMLElements htmlElements_ = new HTMLElements();
 
   /**
    * Stores memoized hash of element -> list of valid subelements.
@@ -63,9 +64,8 @@ public class Html4ElementDescription extends RubyObject
        * the list of elements directly because it's protected.
        */
       for (short c = 0; c < HTMLElements.UNKNOWN; c++) {
-        HTMLElements.Element maybe_sub =
-          HTMLElements.getElement(c);
-        if (maybe_sub.isParent(elem)) {
+        HTMLElements.Element maybe_sub = htmlElements_.getElement(c);
+        if (maybe_sub != null && maybe_sub.isParent(elem)) {
           subs.add(maybe_sub.name);
         }
       }
@@ -82,11 +82,10 @@ public class Html4ElementDescription extends RubyObject
       IRubyObject klazz, IRubyObject name)
   {
 
-    // nekohtml will return an element even for invalid names, see
-    // http://sourceforge.net/p/nekohtml/code/HEAD/tree/trunk/src/org/cyberneko/html/HTMLElements.java#l514
-    // which breaks `test_fetch_nonexistent'
-    HTMLElements.Element elem = HTMLElements.getElement(name.asJavaString(), HTMLElements.NO_SUCH_ELEMENT);
-    if (elem == HTMLElements.NO_SUCH_ELEMENT) {
+    // nekohtml will return an element even for invalid names, which breaks `test_fetch_nonexistent'
+    // see getElement() in HTMLElements.java
+    HTMLElements.Element elem = htmlElements_.getElement(name.asJavaString(), htmlElements_.NO_SUCH_ELEMENT);
+    if (elem == htmlElements_.NO_SUCH_ELEMENT) {
       return context.nil;
     }
 
diff --git a/ext/java/nokogiri/Html4EntityLookup.java b/ext/java/nokogiri/Html4EntityLookup.java
@@ -2,7 +2,7 @@
 
 import static org.jruby.runtime.Helpers.invoke;
 
-import org.cyberneko.html.HTMLEntities;
+import net.sourceforge.htmlunit.cyberneko.HTMLEntitiesParser;
 import org.jruby.Ruby;
 import org.jruby.RubyClass;
 import org.jruby.RubyObject;
@@ -38,8 +38,18 @@ public class Html4EntityLookup extends RubyObject
   {
     Ruby ruby = context.getRuntime();
     String name = key.toString();
-    int val = HTMLEntities.get(name);
-    if (val == -1) { return ruby.getNil(); }
+
+    HTMLEntitiesParser parser = new HTMLEntitiesParser();
+    for (int j = 0 ; j < name.length() ; j++) {
+      if (!parser.parse(name.charAt(j))) {
+        break;
+      }
+    }
+    String match = parser.getMatch();
+
+    if (match == null) { return ruby.getNil(); }
+
+    int val = match.charAt(0);
 
     IRubyObject edClass =
       ruby.getClassFromPath("Nokogiri::HTML4::EntityDescription");
diff --git a/ext/java/nokogiri/Html4SaxParserContext.java b/ext/java/nokogiri/Html4SaxParserContext.java
@@ -9,7 +9,7 @@
 import java.util.regex.Pattern;
 
 import org.apache.xerces.parsers.AbstractSAXParser;
-import org.cyberneko.html.parsers.SAXParser;
+import net.sourceforge.htmlunit.cyberneko.parsers.SAXParser;
 import org.jruby.Ruby;
 import org.jruby.RubyClass;
 import org.jruby.RubyFixnum;
diff --git a/ext/java/nokogiri/NokogiriService.java b/ext/java/nokogiri/NokogiriService.java
@@ -86,7 +86,6 @@ public class NokogiriService implements BasicLibraryService
     RubyModule htmlSaxModule = htmlModule.defineModuleUnder("SAX");
     RubyModule xsltModule = nokogiri.defineModuleUnder("XSLT");
 
-    createJavaLibraryVersionConstants(ruby, nokogiri);
     createNokogiriModule(ruby, nokogiri);
     createSyntaxErrors(ruby, nokogiri, xmlModule);
     RubyClass xmlNode = createXmlModule(ruby, xmlModule);
@@ -97,12 +96,6 @@ public class NokogiriService implements BasicLibraryService
     nokogiri.setInternalVariable("cache", populateNokogiriClassCahce(ruby));
   }
 
-  private void
-  createJavaLibraryVersionConstants(Ruby ruby, RubyModule nokogiri)
-  {
-    nokogiri.defineConstant("NEKO_VERSION", ruby.newString(org.cyberneko.html.Version.getVersion()));
-  }
-
   private void
   createNokogiriModule(Ruby ruby, RubyModule nokogiri)
   {
diff --git a/ext/java/nokogiri/internals/HtmlDomParserContext.java b/ext/java/nokogiri/internals/HtmlDomParserContext.java
@@ -15,8 +15,8 @@
 import org.apache.xerces.xni.XNIException;
 import org.apache.xerces.xni.parser.XMLDocumentFilter;
 import org.apache.xerces.xni.parser.XMLParserConfiguration;
-import org.cyberneko.html.HTMLConfiguration;
-import org.cyberneko.html.filters.DefaultFilter;
+import net.sourceforge.htmlunit.cyberneko.HTMLConfiguration;
+import net.sourceforge.htmlunit.cyberneko.filters.DefaultFilter;
 import org.jruby.Ruby;
 import org.jruby.RubyClass;
 import org.jruby.runtime.ThreadContext;
diff --git a/ext/java/nokogiri/internals/SaveContextVisitor.java b/ext/java/nokogiri/internals/SaveContextVisitor.java
@@ -12,7 +12,7 @@
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
-import org.cyberneko.html.HTMLElements;
+import net.sourceforge.htmlunit.cyberneko.HTMLElements;
 import org.w3c.dom.Attr;
 import org.w3c.dom.CDATASection;
 import org.w3c.dom.Comment;
@@ -81,6 +81,8 @@ public class SaveContextVisitor
   public static final int SUBSETS = 8;
   public static final int EXCLUSIVE = 16;
 
+  private static final HTMLElements htmlElements_ = new HTMLElements();
+
   public
   SaveContextVisitor(int options, CharSequence indent, String encoding, boolean htmlDoc, boolean fragment,
                      int canonicalOpts)
@@ -498,7 +500,7 @@ public class SaveContextVisitor
   private boolean
   isEmpty(String name)
   {
-    HTMLElements.Element element = HTMLElements.getElement(name);
+    HTMLElements.Element element = htmlElements_.getElement(name);
     return element.isEmpty();
   }
 
diff --git a/lib/nokogiri/jruby/dependencies.rb b/lib/nokogiri/jruby/dependencies.rb
@@ -9,7 +9,6 @@
 # should skip loading xml jars. This is because those are in WEB-INF/lib and
 # already set in the classpath.
 unless $LOAD_PATH.to_s.include?("appengine-rack")
-  require "nekohtml.jar"
   require "nekodtd.jar"
 end
 
diff --git a/lib/nokogiri/jruby/nekohtml.jar b/lib/nokogiri/jruby/nekohtml.jar
diff --git a/lib/nokogiri/jruby/net/sourceforge/htmlunit/neko-htmlunit/2.61.0/neko-htmlunit-2.61.0.jar b/lib/nokogiri/jruby/net/sourceforge/htmlunit/neko-htmlunit/2.61.0/neko-htmlunit-2.61.0.jar
diff --git a/lib/nokogiri/jruby/nokogiri_jars.rb b/lib/nokogiri/jruby/nokogiri_jars.rb
@@ -8,6 +8,7 @@
   require 'net/sf/saxon/Saxon-HE/9.6.0-4/Saxon-HE-9.6.0-4.jar'
   require 'xml-apis/xml-apis/1.4.01/xml-apis-1.4.01.jar'
   require 'xalan/serializer/2.7.2/serializer-2.7.2.jar'
+  require 'net/sourceforge/htmlunit/neko-htmlunit/2.61.0/neko-htmlunit-2.61.0.jar'
   require 'isorelax/isorelax/20030108/isorelax-20030108.jar'
 end
 
@@ -18,11 +19,13 @@
   require_jar 'net.sf.saxon', 'Saxon-HE', '9.6.0-4'
   require_jar 'xml-apis', 'xml-apis', '1.4.01'
   require_jar 'xalan', 'serializer', '2.7.2'
+  require_jar 'net.sourceforge.htmlunit', 'neko-htmlunit', '2.61.0'
   require_jar 'isorelax', 'isorelax', '20030108'
 end
 
 # generated by the :vendor_jars rake task
 module Nokogiri
-  JAR_DEPENDENCIES = {"isorelax"=>"isorelax:isorelax:20030108", "jing"=>"nu.validator:jing:20200702VNU", "serializer"=>"xalan:serializer:2.7.2", "xalan"=>"xalan:xalan:2.7.2", "xercesImpl"=>"xerces:xercesImpl:2.12.2", "xml-apis"=>"xml-apis:xml-apis:1.4.01"}.freeze
+  JAR_DEPENDENCIES = {"isorelax"=>"isorelax:isorelax:20030108", "neko-htmlunit"=>"net.sourceforge.htmlunit:neko-htmlunit:2.61.0", "jing"=>"nu.validator:jing:20200702VNU", "serializer"=>"xalan:serializer:2.7.2", "xalan"=>"xalan:xalan:2.7.2", "xercesImpl"=>"xerces:xercesImpl:2.12.2", "xml-apis"=>"xml-apis:xml-apis:1.4.01"}.freeze
   XERCES_VERSION = JAR_DEPENDENCIES["xercesImpl"]
+  NEKO_VERSION = JAR_DEPENDENCIES["neko-htmlunit"]
 end
diff --git a/lib/nokogiri/version/info.rb b/lib/nokogiri/version/info.rb
@@ -169,7 +169,6 @@ def to_hash
           vi["other_libraries"] = Hash[*Nokogiri::OTHER_LIBRARY_VERSIONS.split(/[,:]/)]
         elsif jruby?
           vi["other_libraries"] = {}.tap do |ol|
-            ol["nekohtml"] = Nokogiri::NEKO_VERSION
             Nokogiri::JAR_DEPENDENCIES.each do |k, v|
               ol[k] = v
             end
diff --git a/nokogiri.gemspec b/nokogiri.gemspec
@@ -260,8 +260,8 @@ Gem::Specification.new do |spec|
     "lib/nokogiri/jruby/dependencies.rb",
     "lib/nokogiri/jruby/isorelax/isorelax/20030108/isorelax-20030108.jar",
     "lib/nokogiri/jruby/nekodtd.jar",
-    "lib/nokogiri/jruby/nekohtml.jar",
     "lib/nokogiri/jruby/net/sf/saxon/Saxon-HE/9.6.0-4/Saxon-HE-9.6.0-4.jar",
+    "lib/nokogiri/jruby/net/sourceforge/htmlunit/neko-htmlunit/2.61.0/neko-htmlunit-2.61.0.jar",
     "lib/nokogiri/jruby/nokogiri_jars.rb",
     "lib/nokogiri/jruby/nu/validator/jing/20200702VNU/jing-20200702VNU.jar",
     "lib/nokogiri/jruby/xalan/serializer/2.7.2/serializer-2.7.2.jar",
@@ -326,6 +326,7 @@ Gem::Specification.new do |spec|
     spec.requirements << "jar isorelax, isorelax, 20030108" # https://search.maven.org/artifact/isorelax/isorelax
     # spec.requirements << "jar nekohtml, nekodtd, 0.1.11" # FIXME, we should use Shahid's fork
     # spec.requirements << "jar nekohtml, nekohtml, 1.9.6.2" # FIXME, we should use Shahid's fork
+    spec.requirements << "jar net.sourceforge.htmlunit, neko-htmlunit, 2.61.0"
     spec.requirements << "jar nu.validator, jing, 20200702VNU" # https://search.maven.org/artifact/nu.validator/jing
     spec.requirements << "jar xalan, serializer, 2.7.2" # https://search.maven.org/artifact/xalan/serializer
     spec.requirements << "jar xalan, xalan, 2.7.2" # https://search.maven.org/artifact/xalan/xalan
diff --git a/rakelib/extensions.rake b/rakelib/extensions.rake
@@ -396,6 +396,7 @@ if java?
         module Nokogiri
           JAR_DEPENDENCIES = #{jar_dependencies}.freeze
           XERCES_VERSION = JAR_DEPENDENCIES["xercesImpl"]
+          NEKO_VERSION = JAR_DEPENDENCIES["neko-htmlunit"]
         end
       EOF
     end
diff --git a/scripts/test-gem-file-contents b/scripts/test-gem-file-contents
@@ -271,12 +271,12 @@ describe File.basename(gemfile) do
     it "contains the java jar files" do
       assert_includes(gemfile_contents, "lib/nokogiri/nokogiri.jar")
       assert_includes(gemfile_contents, "lib/nokogiri/jruby/nekodtd.jar")
-      assert_includes(gemfile_contents, "lib/nokogiri/jruby/nekohtml.jar")
 
       actual_jars = gemfile_contents.grep(/.*\.jar$/)
       expected_jars = [
         "isorelax",
         "jing",
+        "neko-htmlunit",
         "serializer",
         "xalan",
         "xercesImpl",
diff --git a/test/html4/test_comments.rb b/test/html4/test_comments.rb
@@ -75,7 +75,7 @@ class TestComment < Nokogiri::TestCase
             it "behaves as if the comment is closed correctly" do # COMPLIANT
               assert_equal 1, subject.children.length
               assert_predicate subject.children.first, :comment?
-              assert_equal "-", subject.children.first.content # curious, potentially non-compliant?
+              assert_equal "", subject.children.first.content
               assert other_div
             end
           end
diff --git a/test/html4/test_element_description.rb b/test/html4/test_element_description.rb
@@ -63,7 +63,7 @@ def test_subelements
         elsif Nokogiri.uses_libxml?
           assert_equal(61, sub_elements.length)
         else
-          refute_empty(sub_elements)
+          assert_equal(105, sub_elements.length)
         end
       end
 
diff --git a/test/test_version.rb b/test/test_version.rb
@@ -32,7 +32,7 @@ def test_version_info_basics
   def test_version_info_for_xerces_and_nekohtml
     skip_unless_jruby("xerces/nekohtml is only used for JRuby")
     assert_equal(Nokogiri::XERCES_VERSION, version_info["other_libraries"]["xercesImpl"])
-    assert_equal(Nokogiri::NEKO_VERSION, version_info["other_libraries"]["nekohtml"])
+    assert_equal(Nokogiri::NEKO_VERSION, version_info["other_libraries"]["neko-htmlunit"])
     refute_nil(version_info["other_libraries"]["isorelax"])
     refute_nil(version_info["other_libraries"]["jing"])
     refute_nil(version_info["other_libraries"]["serializer"])
diff --git a/test/xml/test_node.rb b/test/xml/test_node.rb
@@ -95,7 +95,9 @@ def test_node_context_parsing_of_malformed_html_fragment
           context_node = doc.at_css("div")
           nodeset = context_node.parse("<div </div>")
 
-          assert_equal(1, doc.errors.length)
+          pending_if("nekohtml commit 21286e4 not applied to neko-htmlunit", Nokogiri::VERSION_INFO["other_libraries"]["neko-htmlunit"]) do
+            assert_equal(1, doc.errors.length)
+          end
           assert_equal(1, nodeset.length)
           assert_equal("<div></div>", nodeset.to_s)
           assert_instance_of(Nokogiri::HTML4::Document, nodeset.document)
@@ -107,7 +109,9 @@ def test_node_context_parsing_of_malformed_html_fragment_with_recover_is_correct
           context_node = doc.at_css("div")
           nodeset = context_node.parse("<div </div>", &:recover)
 
-          assert_equal(1, doc.errors.length)
+          pending_if("nekohtml commit 21286e4 not applied to neko-htmlunit", Nokogiri::VERSION_INFO["other_libraries"]["neko-htmlunit"]) do
+            assert_equal(1, doc.errors.length)
+          end
           assert_equal(1, nodeset.length)
           assert_equal("<div></div>", nodeset.to_s)
           assert_instance_of(Nokogiri::HTML4::Document, nodeset.document)
@@ -117,8 +121,10 @@ def test_node_context_parsing_of_malformed_html_fragment_with_recover_is_correct
         def test_node_context_parsing_of_malformed_html_fragment_without_recover_is_not_corrected
           doc = HTML4.parse("<html><body><div></div></body></html>")
           context_node = doc.at_css("div")
-          assert_raises(Nokogiri::XML::SyntaxError) do
-            context_node.parse("<div </div>", &:strict)
+          pending_if("nekohtml commit 21286e4 not applied to neko-htmlunit", Nokogiri::VERSION_INFO["other_libraries"]["neko-htmlunit"]) do
+            assert_raises(Nokogiri::XML::SyntaxError) do
+              context_node.parse("<div </div>", &:strict)
+            end
           end
         end
 
