From c0b6193801e9f2b29fdc68fa68df5834529dba08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mariusz=20=C4=86wikla?= Date: Wed, 11 Nov 2020 15:55:00 +0100 Subject: [PATCH] Fix integer overflow in hash_old --- src/main/java/org/jruby/ext/openssl/X509Name.java | 2 +- src/main/java/org/jruby/ext/openssl/x509store/Name.java | 4 ++-- src/test/ruby/x509/test_x509name.rb | 2 ++ 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/jruby/ext/openssl/X509Name.java b/src/main/java/org/jruby/ext/openssl/X509Name.java index ded0a6c5..8d08e433 100644 --- a/src/main/java/org/jruby/ext/openssl/X509Name.java +++ b/src/main/java/org/jruby/ext/openssl/X509Name.java @@ -633,7 +633,7 @@ public RubyFixnum hash() { @JRubyMethod public RubyFixnum hash_old() { - int hash; + long hash; try { hash = Name.hashOld( getX500Name() ); } diff --git a/src/main/java/org/jruby/ext/openssl/x509store/Name.java b/src/main/java/org/jruby/ext/openssl/x509store/Name.java index 04cf4644..121642b3 100644 --- a/src/main/java/org/jruby/ext/openssl/x509store/Name.java +++ b/src/main/java/org/jruby/ext/openssl/x509store/Name.java @@ -59,12 +59,12 @@ public Name(final X500Name name) { this.name = name; } - public static int hashOld(final X500Name name) throws IOException { + public static long hashOld(final X500Name name) throws IOException { try { final byte[] bytes = name.getEncoded(); MessageDigest md5 = SecurityHelper.getMessageDigest("MD5"); final byte[] digest = md5.digest(bytes); - int result = 0; + long result = 0; result |= digest[3] & 0xff; result <<= 8; result |= digest[2] & 0xff; result <<= 8; result |= digest[1] & 0xff; result <<= 8; diff --git a/src/test/ruby/x509/test_x509name.rb b/src/test/ruby/x509/test_x509name.rb index c312f739..b5a16bac 100644 --- a/src/test/ruby/x509/test_x509name.rb +++ b/src/test/ruby/x509/test_x509name.rb @@ -84,6 +84,8 @@ def test_hash_long_name def test_hash_old name = OpenSSL::X509::Name.new [['CN', 'nobody'], ['DC', 'example']] assert_equal 1460400684, name.hash_old + name = OpenSSL::X509::Name.new([['CN', 'foo'], ['DC', 'bar']]) + assert_equal 3294068023, name.hash_old end end \ No newline at end of file