QUIC APL: Refine domain flag handling

Reviewed-by: Matt Caswell <matt@openssl.org>
Reviewed-by: Neil Horman <nhorman@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/24971)

ssl/quic/quic_impl.c

@@ -4527,6 +4527,18 @@ SSL *ossl_quic_new_domain(SSL_CTX *ctx, uint64_t flags)
 {
     QUIC_DOMAIN *qd = NULL;
     QUIC_ENGINE_ARGS engine_args = {0};
+    uint64_t domain_flags;
+
+    domain_flags = ctx->domain_flags;
+    if ((flags & (SSL_DOMAIN_FLAG_SINGLE_THREAD
+                  | SSL_DOMAIN_FLAG_MULTI_THREAD
+                  | SSL_DOMAIN_FLAG_THREAD_ASSISTED)) != 0)
+        domain_flags = flags;
+    else
+        domain_flags = ctx->domain_flags | flags;
+
+    if (!ossl_adjust_domain_flags(domain_flags, &domain_flags))
+        return NULL;
 
     if ((qd = OPENSSL_zalloc(sizeof(*qd))) == NULL) {
         QUIC_RAISE_NON_NORMAL_ERROR(NULL, ERR_R_CRYPTO_LIB, NULL);
@@ -4545,7 +4557,7 @@ SSL *ossl_quic_new_domain(SSL_CTX *ctx, uint64_t flags)
 #if defined(OPENSSL_THREADS)
     engine_args.mutex   = qd->mutex;
 #endif
-    if (need_notifier_for_domain_flags(ctx->domain_flags))
+    if (need_notifier_for_domain_flags(domain_flags))
         engine_args.reactor_flags |= QUIC_REACTOR_FLAG_USE_NOTIFIER;
 
     if ((qd->engine = ossl_quic_engine_new(&engine_args)) == NULL) {
@@ -4558,6 +4570,7 @@ SSL *ossl_quic_new_domain(SSL_CTX *ctx, uint64_t flags)
                             qd->engine, NULL))
         goto err;
 
+    ossl_quic_obj_set_domain_flags(&qd->obj, domain_flags);
     return &qd->obj.ssl;
 
 err:

ssl/quic/quic_obj_local.h

@@ -327,5 +327,15 @@ ossl_quic_obj_get0_port_leader(const QUIC_OBJ *obj)
         : NULL;
 }
 
+/*
+ * Change the domain flags. Should only be called immediately after
+ * ossl_quic_obj_init().
+ */
+static ossl_inline ossl_unused void
+ossl_quic_obj_set_domain_flags(QUIC_OBJ *obj, uint64_t domain_flags)
+{
+    obj->domain_flags = domain_flags;
+}
+
 # endif
 #endif

ssl/ssl_lib.c

@@ -8020,10 +8020,8 @@ SSL *SSL_new_domain(SSL_CTX *ctx, uint64_t flags)
 #endif
 }
 
-int SSL_CTX_set_domain_flags(SSL_CTX *ctx, uint64_t domain_flags)
+int ossl_adjust_domain_flags(uint64_t domain_flags, uint64_t *p_domain_flags)
 {
-#ifndef OPENSSL_NO_QUIC
-    if (IS_QUIC_CTX(ctx)) {
     if ((domain_flags & ~OSSL_QUIC_SUPPORTED_DOMAIN_FLAGS) != 0) {
         ERR_raise_data(ERR_LIB_SSL, ERR_R_UNSUPPORTED,
                        "unsupported domain flag requested");
@@ -8056,6 +8054,17 @@ int SSL_CTX_set_domain_flags(SSL_CTX *ctx, uint64_t domain_flags)
     }
 # endif
 
+    *p_domain_flags = domain_flags;
+    return 1;
+}
+
+int SSL_CTX_set_domain_flags(SSL_CTX *ctx, uint64_t domain_flags)
+{
+#ifndef OPENSSL_NO_QUIC
+    if (IS_QUIC_CTX(ctx)) {
+        if (!ossl_adjust_domain_flags(domain_flags, &domain_flags))
+            return 0;
+
         ctx->domain_flags = domain_flags;
         return 1;
     }

ssl/ssl_local.h

@@ -2908,6 +2908,9 @@ int ssl_get_md_idx(int md_nid);
 __owur const EVP_MD *ssl_handshake_md(SSL_CONNECTION *s);
 __owur const EVP_MD *ssl_prf_md(SSL_CONNECTION *s);
 
+__owur int ossl_adjust_domain_flags(uint64_t domain_flags,
+                                    uint64_t *p_domain_flags);
+
 /*
  * ssl_log_rsa_client_key_exchange logs |premaster| to the SSL_CTX associated
  * with |ssl|, if logging is enabled. It returns one on success and zero on