reject zero-sized digests in HKDF EVP_PKEY

Reject digests with EVP_MD_size <= 0 when configuring HKDF and when deriving,
preventing divide-by-zero in HKDF_expand if an XOF digest is selected.

We would like to thank Joshua Rogers (https://joshua.hu/) of AISLE Research Team (https://aisle.com/) for reporting this issue.

Author: dkostic

crypto/fipsmodule/evp/p_hkdf.c

@@ -100,6 +100,10 @@ static int pkey_hkdf_derive(EVP_PKEY_CTX *ctx, uint8_t *out, size_t *out_len) {
     OPENSSL_PUT_ERROR(EVP, EVP_R_MISSING_PARAMETERS);
     return 0;
   }
+  if (EVP_MD_size(hctx->md) <= 0) {
+    OPENSSL_PUT_ERROR(EVP, EVP_R_INVALID_OPERATION);
+    return 0;
+  }
   if (hctx->key_len == 0) {
     OPENSSL_PUT_ERROR(EVP, EVP_R_NO_KEY_SET);
     return 0;
@@ -148,7 +152,11 @@ static int pkey_hkdf_ctrl(EVP_PKEY_CTX *ctx, int type, int p1, void *p2) {
       hctx->mode = p1;
       return 1;
     case EVP_PKEY_CTRL_HKDF_MD:
-      hctx->md = p2;
+      if (p2 == NULL || EVP_MD_size((const EVP_MD *)p2) <= 0) {
+        OPENSSL_PUT_ERROR(EVP, EVP_R_INVALID_OPERATION);
+        return 0;
+      }
+      hctx->md = (const EVP_MD *)p2;
       return 1;
     case EVP_PKEY_CTRL_HKDF_KEY: {
       const CBS *key = p2;

crypto/fipsmodule/hkdf/hkdf_test.cc

@@ -579,3 +579,17 @@ static void RunTest(FileTest *t)
 TEST(HKDFTest, HKDFExpandCAVP) {
   FileTestGTest("crypto/evp_extra/kbkdf_expand_tests.txt", RunTest);
 }
+
+TEST(HKDFTest, EVP_PKEY_HKDF_RejectsXOF) {
+  bssl::UniquePtr<EVP_PKEY_CTX> ctx(
+      EVP_PKEY_CTX_new_id(EVP_PKEY_HKDF, nullptr));
+  ASSERT_TRUE(ctx);
+  ASSERT_TRUE(EVP_PKEY_derive_init(ctx.get()));
+
+  // XOF digests should be rejected when setting the digest.
+  EXPECT_FALSE(EVP_PKEY_CTX_set_hkdf_md(ctx.get(), EVP_shake128()));
+  EXPECT_FALSE(EVP_PKEY_CTX_set_hkdf_md(ctx.get(), EVP_shake256()));
+
+  // Non-XOF digests should succeed.
+  EXPECT_TRUE(EVP_PKEY_CTX_set_hkdf_md(ctx.get(), EVP_sha256()));
+}