Add AES-CFB128 optimizations with Intel AVX-512 and VAES

Reviewed-by: Neil Horman <nhorman@openssl.org>
Reviewed-by: Shane Lontis <shane.lontis@oracle.com>
(Merged from https://github.com/openssl/openssl/pull/26902)

CHANGES.md

@@ -127,6 +127,12 @@ OpenSSL 3.6
 
    *Daniel Van Geest (CryptoNext Security)*
 
+ * Added Intel AVX-512 and VAES optimizations for AES-CFB128 algorithms.
+   Encryption performance on large buffers improved by 1.5-1.7x,
+   while decryption speed increased by 20-23x.
+
+   *Adrian Stanciu*
+
 OpenSSL 3.5
 -----------
 

crypto/aes/build.info

@@ -10,7 +10,7 @@ IF[{- !$disabled{asm} -}]
   $AESASM_x86_64=\
         aes-x86_64.s vpaes-x86_64.s bsaes-x86_64.s aesni-x86_64.s \
         aesni-sha1-x86_64.s aesni-sha256-x86_64.s aesni-mb-x86_64.s \
-        aesni-xts-avx512.s
+        aesni-xts-avx512.s aes-cfb-avx512.s
   $AESDEF_x86_64=AES_ASM VPAES_ASM BSAES_ASM
 
   $AESASM_ia64=aes_core.c aes_cbc.c aes-ia64.s
@@ -107,6 +107,7 @@ GENERATE[aes-x86_64.s]=asm/aes-x86_64.pl
 GENERATE[vpaes-x86_64.s]=asm/vpaes-x86_64.pl
 GENERATE[bsaes-x86_64.s]=asm/bsaes-x86_64.pl
 GENERATE[aesni-x86_64.s]=asm/aesni-x86_64.pl
+GENERATE[aes-cfb-avx512.s]=asm/aes-cfb-avx512.pl
 GENERATE[aesni-sha1-x86_64.s]=asm/aesni-sha1-x86_64.pl
 GENERATE[aesni-sha256-x86_64.s]=asm/aesni-sha256-x86_64.pl
 GENERATE[aesni-mb-x86_64.s]=asm/aesni-mb-x86_64.pl

crypto/perlasm/x86_64-xlate.pl

@@ -440,6 +440,7 @@ my %globals;
 	    ($self->{asterisk})				&& ($sz="q") ||
 	    ($mnemonic =~ /^v?mov([qd])$/)		&& ($sz=$1)  ||
 	    ($mnemonic =~ /^v?pinsr([qdwb])$/)		&& ($sz=$1)  ||
+	    ($mnemonic =~ /^vbroadcasti32x4$/)		&& ($sz="x") ||
 	    ($mnemonic =~ /^vpbroadcast([qdwb])$/)	&& ($sz=$1)  ||
 	    ($mnemonic =~ /^v(?!perm)[a-z]+[fi]128$/)	&& ($sz="x");
 

include/crypto/aes_platform.h

@@ -208,6 +208,14 @@ int aesni_set_encrypt_key(const unsigned char *userKey, int bits,
 int aesni_set_decrypt_key(const unsigned char *userKey, int bits,
                           AES_KEY *key);
 
+void ossl_aes_cfb128_vaes_enc(const unsigned char *in, unsigned char *out,
+                              size_t len, const AES_KEY *ks,
+                              const unsigned char ivec[16], ossl_ssize_t *num);
+void ossl_aes_cfb128_vaes_dec(const unsigned char *in, unsigned char *out,
+                              size_t len, const AES_KEY *ks,
+                              const unsigned char ivec[16], ossl_ssize_t *num);
+int ossl_aes_cfb128_vaes_eligible(void);
+
 void aesni_encrypt(const unsigned char *in, unsigned char *out,
                    const AES_KEY *key);
 void aesni_decrypt(const unsigned char *in, unsigned char *out,

providers/implementations/ciphers/cipher_aes_cfb_hw_aesni.inc

@@ -8,14 +8,56 @@
  */
 
 /*-
- * AES-NI support for AES mode cfb.
+ * AES-NI and VAES support for AES CFB mode.
  * This file is included by cipher_aes_cfb_hw.c
  */
 
+#if (defined(__x86_64) || defined(__x86_64__) || defined(_M_AMD64) || defined(_M_X64))
+    #define cipher_hw_vaes_cfb128 aes_cfb128_vaes_encdec_wrapper
+#else
+    #define cipher_hw_vaes_cfb128 ossl_cipher_hw_generic_cfb128
+    int ossl_aes_cfb128_vaes_eligible() {
+        return 0;
+    }
+#endif
+#define cipher_hw_vaes_cfb8    ossl_cipher_hw_generic_cfb8
+#define cipher_hw_vaes_cfb1    ossl_cipher_hw_generic_cfb1
+
 #define cipher_hw_aesni_cfb128 ossl_cipher_hw_generic_cfb128
 #define cipher_hw_aesni_cfb8   ossl_cipher_hw_generic_cfb8
 #define cipher_hw_aesni_cfb1   ossl_cipher_hw_generic_cfb1
 
+static int ossl_aes_cfb8_vaes_eligible(void) { return 0; }
+static int ossl_aes_cfb1_vaes_eligible(void) { return 0; }
+
+/* active in 64-bit builds when AES-NI, AVX512F, and VAES are detected */
+static int aes_cfb128_vaes_encdec_wrapper(
+    PROV_CIPHER_CTX* dat,
+    unsigned char *out,
+    const unsigned char *in,
+    size_t len)
+{
+    ossl_ssize_t num;
+
+    num = (ossl_ssize_t)dat->num;
+
+    if (num < 0) {
+        /* behavior from CRYPTO_cfb128_encrypt */
+        dat->num = -1;
+        return 1;
+    }
+
+    if (dat->enc)
+        ossl_aes_cfb128_vaes_enc(in, out, len, dat->ks, dat->iv, &num);
+    else
+        ossl_aes_cfb128_vaes_dec(in, out, len, dat->ks, dat->iv, &num);
+
+    dat->num = (int)num;
+
+    return 1;
+}
+ 
+/* generates AES round keys for AES-NI and VAES implementations */
 static int cipher_hw_aesni_initkey(PROV_CIPHER_CTX *dat,
                                    const unsigned char *key, size_t keylen)
 {
@@ -43,7 +85,15 @@ static const PROV_CIPHER_HW aesni_##mode = {                                   \
     cipher_hw_aesni_initkey,                                                   \
     cipher_hw_aesni_##mode,                                                    \
     cipher_hw_aes_copyctx                                                      \
+};                                                                             \
+static const PROV_CIPHER_HW vaes_##mode = {                                    \
+    cipher_hw_aesni_initkey,                                                   \
+    cipher_hw_vaes_##mode,                                                     \
+    cipher_hw_aes_copyctx                                                      \
 };
 #define PROV_CIPHER_HW_select(mode)                                            \
-if (AESNI_CAPABLE)                                                             \
-    return &aesni_##mode;
+if (AESNI_CAPABLE) {                                                           \
+    if (ossl_aes_##mode##_vaes_eligible())                                     \
+        return &vaes_##mode;                                                   \
+    return &aesni_##mode;                                                      \
+}