QUIC APL: Refine domain flag handling

Reviewed-by: Matt Caswell <matt@openssl.org>
Reviewed-by: Neil Horman <nhorman@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/24971)

ssl/quic/quic_impl.c

@@ -4527,6 +4527,18 @@ SSL *ossl_quic_new_domain(SSL_CTX *ctx, uint64_t flags)
 {
     QUIC_DOMAIN *qd = NULL;
     QUIC_ENGINE_ARGS engine_args = {0};
+    uint64_t domain_flags;
+
+    domain_flags = ctx->domain_flags;
+    if ((flags & (SSL_DOMAIN_FLAG_SINGLE_THREAD
+                  | SSL_DOMAIN_FLAG_MULTI_THREAD
+                  | SSL_DOMAIN_FLAG_THREAD_ASSISTED)) != 0)
+        domain_flags = flags;
+    else
+        domain_flags = ctx->domain_flags | flags;
+
+    if (!ossl_adjust_domain_flags(domain_flags, &domain_flags))
+        return NULL;
 
     if ((qd = OPENSSL_zalloc(sizeof(*qd))) == NULL) {
         QUIC_RAISE_NON_NORMAL_ERROR(NULL, ERR_R_CRYPTO_LIB, NULL);
@@ -4545,7 +4557,7 @@ SSL *ossl_quic_new_domain(SSL_CTX *ctx, uint64_t flags)
 #if defined(OPENSSL_THREADS)
     engine_args.mutex   = qd->mutex;
 #endif
-    if (need_notifier_for_domain_flags(ctx->domain_flags))
+    if (need_notifier_for_domain_flags(domain_flags))
         engine_args.reactor_flags |= QUIC_REACTOR_FLAG_USE_NOTIFIER;
 
     if ((qd->engine = ossl_quic_engine_new(&engine_args)) == NULL) {
@@ -4558,6 +4570,7 @@ SSL *ossl_quic_new_domain(SSL_CTX *ctx, uint64_t flags)
                             qd->engine, NULL))
         goto err;
 
+    ossl_quic_obj_set_domain_flags(&qd->obj, domain_flags);
     return &qd->obj.ssl;
 
 err:

ssl/quic/quic_obj_local.h

@@ -327,5 +327,15 @@ ossl_quic_obj_get0_port_leader(const QUIC_OBJ *obj)
         : NULL;
 }
 
+/*
+ * Change the domain flags. Should only be called immediately after
+ * ossl_quic_obj_init().
+ */
+static ossl_inline ossl_unused void
+ossl_quic_obj_set_domain_flags(QUIC_OBJ *obj, uint64_t domain_flags)
+{
+    obj->domain_flags = domain_flags;
+}
+
 # endif
 #endif

ssl/ssl_lib.c

@@ -8020,41 +8020,50 @@ SSL *SSL_new_domain(SSL_CTX *ctx, uint64_t flags)
 #endif
 }
 
+int ossl_adjust_domain_flags(uint64_t domain_flags, uint64_t *p_domain_flags)
+{
+    if ((domain_flags & ~OSSL_QUIC_SUPPORTED_DOMAIN_FLAGS) != 0) {
+        ERR_raise_data(ERR_LIB_SSL, ERR_R_UNSUPPORTED,
+                       "unsupported domain flag requested");
+        return 0;
+    }
+
+    if ((domain_flags & SSL_DOMAIN_FLAG_THREAD_ASSISTED) != 0)
+        domain_flags |= SSL_DOMAIN_FLAG_MULTI_THREAD;
+
+    if ((domain_flags & (SSL_DOMAIN_FLAG_MULTI_THREAD
+                         | SSL_DOMAIN_FLAG_SINGLE_THREAD)) == 0)
+        domain_flags |= SSL_DOMAIN_FLAG_MULTI_THREAD;
+
+    if ((domain_flags & SSL_DOMAIN_FLAG_SINGLE_THREAD) != 0
+        && (domain_flags & SSL_DOMAIN_FLAG_MULTI_THREAD) != 0) {
+        ERR_raise_data(ERR_LIB_SSL, ERR_R_PASSED_INVALID_ARGUMENT,
+                       "mutually exclusive domain flags specified");
+        return 0;
+    }
+
+    /*
+     * Note: We treat MULTI_THREAD as a no-op in non-threaded builds, but
+     * not THREAD_ASSISTED.
+     */
+# ifndef OPENSSL_THREADS
+    if ((domain_flags & SSL_DOMAIN_FLAG_THREAD_ASSISTED) != 0) {
+        ERR_raise_data(ERR_LIB_SSL, ERR_R_UNSUPPORTED,
+                       "thread assisted mode not available in this build");
+        return 0;
+    }
+# endif
+
+    *p_domain_flags = domain_flags;
+    return 1;
+}
+
 int SSL_CTX_set_domain_flags(SSL_CTX *ctx, uint64_t domain_flags)
 {
 #ifndef OPENSSL_NO_QUIC
     if (IS_QUIC_CTX(ctx)) {
-        if ((domain_flags & ~OSSL_QUIC_SUPPORTED_DOMAIN_FLAGS) != 0) {
+        if (!ossl_adjust_domain_flags(domain_flags, &domain_flags))
-            ERR_raise_data(ERR_LIB_SSL, ERR_R_UNSUPPORTED,
-                           "unsupported domain flag requested");
             return 0;
-        }
-
-        if ((domain_flags & SSL_DOMAIN_FLAG_THREAD_ASSISTED) != 0)
-            domain_flags |= SSL_DOMAIN_FLAG_MULTI_THREAD;
-
-        if ((domain_flags & (SSL_DOMAIN_FLAG_MULTI_THREAD
-                             | SSL_DOMAIN_FLAG_SINGLE_THREAD)) == 0)
-            domain_flags |= SSL_DOMAIN_FLAG_MULTI_THREAD;
-
-        if ((domain_flags & SSL_DOMAIN_FLAG_SINGLE_THREAD) != 0
-            && (domain_flags & SSL_DOMAIN_FLAG_MULTI_THREAD) != 0) {
-            ERR_raise_data(ERR_LIB_SSL, ERR_R_PASSED_INVALID_ARGUMENT,
-                           "mutually exclusive domain flags specified");
-            return 0;
-        }
-
-        /*
-         * Note: We treat MULTI_THREAD as a no-op in non-threaded builds, but
-         * not THREAD_ASSISTED.
-         */
-# ifndef OPENSSL_THREADS
-        if ((domain_flags & SSL_DOMAIN_FLAG_THREAD_ASSISTED) != 0) {
-            ERR_raise_data(ERR_LIB_SSL, ERR_R_UNSUPPORTED,
-                           "thread assisted mode not available in this build");
-            return 0;
-        }
-# endif
 
         ctx->domain_flags = domain_flags;
         return 1;

ssl/ssl_local.h

@@ -2908,6 +2908,9 @@ int ssl_get_md_idx(int md_nid);
 __owur const EVP_MD *ssl_handshake_md(SSL_CONNECTION *s);
 __owur const EVP_MD *ssl_prf_md(SSL_CONNECTION *s);
 
+__owur int ossl_adjust_domain_flags(uint64_t domain_flags,
+                                    uint64_t *p_domain_flags);
+
 /*
  * ssl_log_rsa_client_key_exchange logs |premaster| to the SSL_CTX associated
  * with |ssl|, if logging is enabled. It returns one on success and zero on