Make DH_check_pub_key() and DH_generate_key() safer yet

We already check for an excessively large P in DH_generate_key(), but not in
DH_check_pub_key(), and none of them check for an excessively large Q.

This change adds all the missing excessive size checks of P and Q.

It's to be noted that behaviours surrounding excessively sized P and Q
differ.  DH_check() raises an error on the excessively sized P, but only
sets a flag for the excessively sized Q.  This behaviour is mimicked in
DH_check_pub_key().

Reviewed-by: Tomas Mraz <tomas@openssl.org>
Reviewed-by: Matt Caswell <matt@openssl.org>
Reviewed-by: Hugo Landau <hlandau@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/22518)

crypto/dh/dh_check.c

@@ -249,6 +249,18 @@ int DH_check_pub_key_ex(const DH *dh, const BIGNUM *pub_key)
  */
 int DH_check_pub_key(const DH *dh, const BIGNUM *pub_key, int *ret)
 {
+    /* Don't do any checks at all with an excessively large modulus */
+    if (BN_num_bits(dh->params.p) > OPENSSL_DH_CHECK_MAX_MODULUS_BITS) {
+        ERR_raise(ERR_LIB_DH, DH_R_MODULUS_TOO_LARGE);
+        *ret = DH_MODULUS_TOO_LARGE | DH_CHECK_PUBKEY_INVALID;
+        return 0;
+    }
+
+    if (dh->params.q != NULL && BN_ucmp(dh->params.p, dh->params.q) < 0) {
+        *ret |= DH_CHECK_INVALID_Q_VALUE | DH_CHECK_PUBKEY_INVALID;
+        return 1;
+    }
+
     return ossl_ffc_validate_public_key(&dh->params, pub_key, ret);
 }
 

crypto/dh/dh_err.c

@@ -1,6 +1,6 @@
 /*
  * Generated by util/mkerr.pl DO NOT EDIT
- * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved.
+ * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved.
  *
  * Licensed under the Apache License 2.0 (the "License").  You may not use
  * this file except in compliance with the License.  You can obtain a copy
@@ -54,6 +54,7 @@ static const ERR_STRING_DATA DH_str_reasons[] = {
     {ERR_PACK(ERR_LIB_DH, 0, DH_R_PARAMETER_ENCODING_ERROR),
     "parameter encoding error"},
     {ERR_PACK(ERR_LIB_DH, 0, DH_R_PEER_KEY_ERROR), "peer key error"},
+    {ERR_PACK(ERR_LIB_DH, 0, DH_R_Q_TOO_LARGE), "q too large"},
     {ERR_PACK(ERR_LIB_DH, 0, DH_R_SHARED_INFO_ERROR), "shared info error"},
     {ERR_PACK(ERR_LIB_DH, 0, DH_R_UNABLE_TO_CHECK_GENERATOR),
     "unable to check generator"},

crypto/dh/dh_key.c

@@ -49,6 +49,12 @@ int ossl_dh_compute_key(unsigned char *key, const BIGNUM *pub_key, DH *dh)
         goto err;
     }
 
+    if (dh->params.q != NULL
+        && BN_num_bits(dh->params.q) > OPENSSL_DH_MAX_MODULUS_BITS) {
+        ERR_raise(ERR_LIB_DH, DH_R_Q_TOO_LARGE);
+        goto err;
+    }
+
     if (BN_num_bits(dh->params.p) < DH_MIN_MODULUS_BITS) {
         ERR_raise(ERR_LIB_DH, DH_R_MODULUS_TOO_SMALL);
         return 0;
@@ -267,6 +273,12 @@ static int generate_key(DH *dh)
         return 0;
     }
 
+    if (dh->params.q != NULL
+        && BN_num_bits(dh->params.q) > OPENSSL_DH_MAX_MODULUS_BITS) {
+        ERR_raise(ERR_LIB_DH, DH_R_Q_TOO_LARGE);
+        return 0;
+    }
+
     if (BN_num_bits(dh->params.p) < DH_MIN_MODULUS_BITS) {
         ERR_raise(ERR_LIB_DH, DH_R_MODULUS_TOO_SMALL);
         return 0;

crypto/err/openssl.txt

@@ -513,6 +513,7 @@ DH_R_NO_PARAMETERS_SET:107:no parameters set
 DH_R_NO_PRIVATE_VALUE:100:no private value
 DH_R_PARAMETER_ENCODING_ERROR:105:parameter encoding error
 DH_R_PEER_KEY_ERROR:111:peer key error
+DH_R_Q_TOO_LARGE:130:q too large
 DH_R_SHARED_INFO_ERROR:113:shared info error
 DH_R_UNABLE_TO_CHECK_GENERATOR:121:unable to check generator
 DSA_R_BAD_FFC_PARAMETERS:114:bad ffc parameters

include/crypto/dherr.h

@@ -1,6 +1,6 @@
 /*
  * Generated by util/mkerr.pl DO NOT EDIT
- * Copyright 2020-2021 The OpenSSL Project Authors. All Rights Reserved.
+ * Copyright 2020-2023 The OpenSSL Project Authors. All Rights Reserved.
  *
  * Licensed under the Apache License 2.0 (the "License").  You may not use
  * this file except in compliance with the License.  You can obtain a copy

include/openssl/dh.h

@@ -144,7 +144,7 @@ DECLARE_ASN1_ITEM(DHparams)
 #   define DH_GENERATOR_3          3
 #   define DH_GENERATOR_5          5
 
-/* DH_check error codes */
+/* DH_check error codes, some of them shared with DH_check_pub_key */
 /*
  * NB: These values must align with the equivalently named macros in
  * internal/ffc.h.
@@ -154,10 +154,10 @@ DECLARE_ASN1_ITEM(DHparams)
 #   define DH_UNABLE_TO_CHECK_GENERATOR    0x04
 #   define DH_NOT_SUITABLE_GENERATOR       0x08
 #   define DH_CHECK_Q_NOT_PRIME            0x10
-#   define DH_CHECK_INVALID_Q_VALUE        0x20
+#   define DH_CHECK_INVALID_Q_VALUE        0x20 /* +DH_check_pub_key */
 #   define DH_CHECK_INVALID_J_VALUE        0x40
 #   define DH_MODULUS_TOO_SMALL            0x80
-#   define DH_MODULUS_TOO_LARGE            0x100
+#   define DH_MODULUS_TOO_LARGE            0x100 /* +DH_check_pub_key */
 
 /* DH_check_pub_key error codes */
 #   define DH_CHECK_PUBKEY_TOO_SMALL       0x01

include/openssl/dherr.h

@@ -1,6 +1,6 @@
 /*
  * Generated by util/mkerr.pl DO NOT EDIT
- * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved.
+ * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved.
  *
  * Licensed under the Apache License 2.0 (the "License").  You may not use
  * this file except in compliance with the License.  You can obtain a copy
@@ -50,6 +50,7 @@
 #  define DH_R_NO_PRIVATE_VALUE                            100
 #  define DH_R_PARAMETER_ENCODING_ERROR                    105
 #  define DH_R_PEER_KEY_ERROR                              111
+#  define DH_R_Q_TOO_LARGE                                 130
 #  define DH_R_SHARED_INFO_ERROR                           113
 #  define DH_R_UNABLE_TO_CHECK_GENERATOR                   121