Add DTLS_get_data_mtu() function

We add ssl_cipher_get_overhead() as an internal function, to avoid
having too much ciphersuite-specific knowledge in DTLS_get_data_mtu()
itself. It's going to need adjustment for TLSv1.3... but then again, so
is fairly much *all* of the SSL_CIPHER handling. This bit is in the noise.

Reviewed-by: Rich Salz <rsalz@openssl.org>
Reviewed-by: Matt Caswell <matt@openssl.org>

include/openssl/ssl.h

@@ -1630,6 +1630,8 @@ __owur const SSL_METHOD *DTLS_method(void); /* DTLS 1.0 and 1.2 */
 __owur const SSL_METHOD *DTLS_server_method(void); /* DTLS 1.0 and 1.2 */
 __owur const SSL_METHOD *DTLS_client_method(void); /* DTLS 1.0 and 1.2 */
 
+__owur size_t DTLS_get_data_mtu(const SSL *s);
+
 __owur STACK_OF(SSL_CIPHER) *SSL_get_ciphers(const SSL *s);
 __owur STACK_OF(SSL_CIPHER) *SSL_CTX_get_ciphers(const SSL_CTX *ctx);
 __owur STACK_OF(SSL_CIPHER) *SSL_get_client_ciphers(const SSL *s);

ssl/d1_lib.c

@@ -1088,3 +1088,39 @@ unsigned int dtls1_min_mtu(SSL *s)
 {
     return dtls1_link_min_mtu() - BIO_dgram_get_mtu_overhead(SSL_get_wbio(s));
 }
+
+size_t DTLS_get_data_mtu(const SSL *s)
+{
+    size_t mac_overhead, int_overhead, blocksize, ext_overhead;
+    const SSL_CIPHER *ciph = SSL_get_current_cipher(s);
+    size_t mtu = s->d1->mtu;
+
+    if (ciph == NULL)
+        return 0;
+
+    if (!ssl_cipher_get_overhead(ciph, &mac_overhead, &int_overhead,
+                                 &blocksize, &ext_overhead))
+        return 0;
+
+    if (SSL_USE_ETM(s))
+        ext_overhead += mac_overhead;
+    else
+        int_overhead += mac_overhead;
+
+    /* Subtract external overhead (e.g. IV/nonce, separate MAC) */
+    if (ext_overhead + DTLS1_RT_HEADER_LENGTH >= mtu)
+        return 0;
+    mtu -= ext_overhead + DTLS1_RT_HEADER_LENGTH;
+
+    /* Round encrypted payload down to cipher block size (for CBC etc.)
+     * No check for overflow since 'mtu % blocksize' cannot exceed mtu. */
+    if (blocksize)
+        mtu -= (mtu % blocksize);
+
+    /* Subtract internal overhead (e.g. CBC padding len byte) */
+    if (int_overhead >= mtu)
+        return 0;
+    mtu -= int_overhead;
+
+    return mtu;
+}

ssl/ssl_ciph.c

@@ -1947,3 +1947,55 @@ int SSL_CIPHER_is_aead(const SSL_CIPHER *c)
 {
     return (c->algorithm_mac & SSL_AEAD) ? 1 : 0;
 }
+
+int ssl_cipher_get_overhead(const SSL_CIPHER *c, size_t *mac_overhead,
+                            size_t *int_overhead, size_t *blocksize,
+                            size_t *ext_overhead)
+{
+    size_t mac = 0, in = 0, blk = 0, out = 0;
+
+    /* Some hard-coded numbers for the CCM/Poly1305 MAC overhead
+     * because there are no handy #defines for those. */
+    if (c->algorithm_enc & SSL_AESGCM) {
+        out = EVP_GCM_TLS_EXPLICIT_IV_LEN + EVP_GCM_TLS_TAG_LEN;
+    } else if (c->algorithm_enc & (SSL_AES128CCM | SSL_AES256CCM)) {
+        out = EVP_CCM_TLS_EXPLICIT_IV_LEN + 16;
+    } else if (c->algorithm_enc & (SSL_AES128CCM8 | SSL_AES256CCM8)) {
+        out = EVP_CCM_TLS_EXPLICIT_IV_LEN + 8;
+    } else if (c->algorithm_enc & SSL_CHACHA20POLY1305) {
+        out = 16;
+    } else if (c->algorithm_mac & SSL_AEAD) {
+        /* We're supposed to have handled all the AEAD modes above */
+        return 0;
+    } else {
+        /* Non-AEAD modes. Calculate MAC/cipher overhead separately */
+        int digest_nid = SSL_CIPHER_get_digest_nid(c);
+        const EVP_MD *e_md = EVP_get_digestbynid(digest_nid);
+
+        if (e_md == NULL)
+            return 0;
+
+        mac = EVP_MD_size(e_md);
+        if (c->algorithm_enc != SSL_eNULL) {
+            int cipher_nid = SSL_CIPHER_get_cipher_nid(c);
+            const EVP_CIPHER *e_ciph = EVP_get_cipherbynid(cipher_nid);
+
+            /* If it wasn't AEAD or SSL_eNULL, we expect it to be a
+               known CBC cipher. */
+            if (e_ciph == NULL ||
+                EVP_CIPHER_mode(e_ciph) != EVP_CIPH_CBC_MODE)
+                return 0;
+
+            in = 1; /* padding length byte */
+            out = EVP_CIPHER_iv_length(e_ciph);
+            blk = EVP_CIPHER_block_size(e_ciph);
+        }
+    }
+
+    *mac_overhead = mac;
+    *int_overhead = in;
+    *blocksize = blk;
+    *ext_overhead = out;
+
+    return 1;
+}

ssl/ssl_locl.h

@@ -1817,6 +1817,9 @@ __owur int ssl_cipher_get_evp(const SSL_SESSION *s, const EVP_CIPHER **enc,
                               const EVP_MD **md, int *mac_pkey_type,
                               int *mac_secret_size, SSL_COMP **comp,
                               int use_etm);
+__owur int ssl_cipher_get_overhead(const SSL_CIPHER *c, size_t *mac_overhead,
+                                   size_t *int_overhead, size_t *blocksize,
+                                   size_t *ext_overhead);
 __owur int ssl_cipher_get_cert_index(const SSL_CIPHER *c);
 __owur const SSL_CIPHER *ssl_get_cipher_by_char(SSL *ssl,
                                                 const unsigned char *ptr);

util/libssl.num

@@ -404,3 +404,4 @@ SSL_SESSION_get0_cipher                 404	1_1_0	EXIST::FUNCTION:
 SSL_SESSION_get0_id_context             405	1_1_0	EXIST::FUNCTION:
 SSL_SESSION_set1_id                     406	1_1_0	EXIST::FUNCTION:
 SSL_CTX_set1_cert_store                 407	1_1_1	EXIST::FUNCTION:
+DTLS_get_data_mtu                       408	1_1_1	EXIST::FUNCTION: