Make libssl async aware

The following entry points have been made async aware:
SSL_accept
SSL_read
SSL_write

Also added is a new mode - SSL_MODE_ASYNC. Calling the above functions with
the async mode enabled will initiate a new async job. If an async pause is
encountered whilst executing the job (such as for example if using SHA1/RSA
with the Dummy Async engine), then the above functions return with
SSL_WANT_ASYNC. Calling the functions again (with exactly the same args
as per non-blocking IO), will resume the job where it left off.

Reviewed-by: Rich Salz <rsalz@openssl.org>

crypto/ct/Makefile

@@ -70,7 +70,8 @@ ct_err.o: ../../include/openssl/opensslv.h ../../include/openssl/ossl_typ.h
 ct_err.o: ../../include/openssl/safestack.h ../../include/openssl/stack.h
 ct_err.o: ../../include/openssl/symhacks.h ../include/internal/ct_int.h
 ct_err.o: ct_err.c
-ct_lib.o: ../../e_os.h ../../include/openssl/asn1.h ../../include/openssl/bio.h
+ct_lib.o: ../../e_os.h ../../include/openssl/asn1.h
+ct_lib.o: ../../include/openssl/async.h ../../include/openssl/bio.h
 ct_lib.o: ../../include/openssl/bn.h ../../include/openssl/buffer.h
 ct_lib.o: ../../include/openssl/comp.h ../../include/openssl/crypto.h
 ct_lib.o: ../../include/openssl/dsa.h ../../include/openssl/dtls1.h

include/openssl/ssl.h

@@ -490,6 +490,10 @@ typedef int (*custom_ext_parse_cb) (SSL *s, unsigned int ext_type,
  * draft-ietf-tls-downgrade-scsv-00.
  */
 # define SSL_MODE_SEND_FALLBACK_SCSV 0x00000080U
+/*
+ * Support Asynchronous operation
+ */
+# define SSL_MODE_ASYNC 0x00000100U
 
 /* Cert related flags */
 /*
@@ -874,16 +878,18 @@ __owur int SSL_CTX_add_server_custom_ext(SSL_CTX *ctx, unsigned int ext_type,
 
 __owur int SSL_extension_supported(unsigned int ext_type);
 
-# define SSL_NOTHING     1
-# define SSL_WRITING     2
-# define SSL_READING     3
-# define SSL_X509_LOOKUP 4
+# define SSL_NOTHING            1
+# define SSL_WRITING            2
+# define SSL_READING            3
+# define SSL_X509_LOOKUP        4
+# define SSL_ASYNC_PAUSED       5
 
 /* These will only be used when doing non-blocking IO */
 # define SSL_want_nothing(s)     (SSL_want(s) == SSL_NOTHING)
 # define SSL_want_read(s)        (SSL_want(s) == SSL_READING)
 # define SSL_want_write(s)       (SSL_want(s) == SSL_WRITING)
 # define SSL_want_x509_lookup(s) (SSL_want(s) == SSL_X509_LOOKUP)
+# define SSL_want_async(s)       (SSL_want(s) == SSL_ASYNC_PAUSED)
 
 # define SSL_MAC_FLAG_READ_MAC_STREAM 1
 # define SSL_MAC_FLAG_WRITE_MAC_STREAM 2
@@ -1102,6 +1108,7 @@ DECLARE_PEM_rw(SSL_SESSION, SSL_SESSION)
 # define SSL_ERROR_ZERO_RETURN           6
 # define SSL_ERROR_WANT_CONNECT          7
 # define SSL_ERROR_WANT_ACCEPT           8
+# define SSL_ERROR_WANT_ASYNC            9
 # define SSL_CTRL_NEED_TMP_RSA                   1
 # define SSL_CTRL_SET_TMP_RSA                    2
 # define SSL_CTRL_SET_TMP_DH                     3
@@ -1564,6 +1571,7 @@ __owur char *SSL_get_srp_userinfo(SSL *s);
 
 void SSL_certs_clear(SSL *s);
 void SSL_free(SSL *ssl);
+__owur int SSL_waiting_for_async(SSL *s);
 __owur int SSL_accept(SSL *ssl);
 __owur int SSL_connect(SSL *ssl);
 __owur int SSL_read(SSL *ssl, void *buf, int num);
@@ -2024,6 +2032,7 @@ void ERR_load_SSL_strings(void);
 # define SSL_F_SSL3_SETUP_WRITE_BUFFER                    291
 # define SSL_F_SSL3_WRITE_BYTES                           158
 # define SSL_F_SSL3_WRITE_PENDING                         159
+# define SSL_F_SSL_ACCEPT                                 351
 # define SSL_F_SSL_ADD_CERT_CHAIN                         316
 # define SSL_F_SSL_ADD_CERT_TO_BUF                        319
 # define SSL_F_SSL_ADD_CLIENTHELLO_RENEGOTIATE_EXT        298
@@ -2266,6 +2275,7 @@ void ERR_load_SSL_strings(void);
 # define SSL_R_ERROR_IN_RECEIVED_CIPHER_LIST              151
 # define SSL_R_EXCESSIVE_MESSAGE_SIZE                     152
 # define SSL_R_EXTRA_DATA_IN_MESSAGE                      153
+# define SSL_R_FAILED_TO_INIT_ASYNC                       405
 # define SSL_R_FRAGMENTED_CLIENT_HELLO                    401
 # define SSL_R_GOT_A_FIN_BEFORE_A_CCS                     154
 # define SSL_R_GOT_NEXT_PROTO_BEFORE_A_CCS                355

ssl/ssl_err.c

@@ -190,6 +190,7 @@ static ERR_STRING_DATA SSL_str_functs[] = {
     {ERR_FUNC(SSL_F_SSL3_SETUP_WRITE_BUFFER), "ssl3_setup_write_buffer"},
     {ERR_FUNC(SSL_F_SSL3_WRITE_BYTES), "ssl3_write_bytes"},
     {ERR_FUNC(SSL_F_SSL3_WRITE_PENDING), "ssl3_write_pending"},
+    {ERR_FUNC(SSL_F_SSL_ACCEPT), "SSL_accept"},
     {ERR_FUNC(SSL_F_SSL_ADD_CERT_CHAIN), "ssl_add_cert_chain"},
     {ERR_FUNC(SSL_F_SSL_ADD_CERT_TO_BUF), "ssl_add_cert_to_buf"},
     {ERR_FUNC(SSL_F_SSL_ADD_CLIENTHELLO_RENEGOTIATE_EXT),
@@ -521,6 +522,7 @@ static ERR_STRING_DATA SSL_str_reasons[] = {
      "error in received cipher list"},
     {ERR_REASON(SSL_R_EXCESSIVE_MESSAGE_SIZE), "excessive message size"},
     {ERR_REASON(SSL_R_EXTRA_DATA_IN_MESSAGE), "extra data in message"},
+    {ERR_REASON(SSL_R_FAILED_TO_INIT_ASYNC), "failed to init async"},
     {ERR_REASON(SSL_R_FRAGMENTED_CLIENT_HELLO), "fragmented client hello"},
     {ERR_REASON(SSL_R_GOT_A_FIN_BEFORE_A_CCS), "got a fin before a ccs"},
     {ERR_REASON(SSL_R_GOT_NEXT_PROTO_BEFORE_A_CCS),

ssl/ssl_lib.c

@@ -158,6 +158,7 @@
 #ifndef OPENSSL_NO_ENGINE
 # include <openssl/engine.h>
 #endif
+#include <openssl/async.h>
 
 const char SSL_version_str[] = OPENSSL_VERSION_TEXT;
 
@@ -186,6 +187,12 @@ SSL3_ENC_METHOD ssl3_undef_enc_method = {
              int use_context))ssl_undefined_function,
 };
 
+struct ssl_async_args {
+    SSL *s;
+    void *buf;
+    int num;
+};
+
 static void clear_ciphers(SSL *s)
 {
     /* clear the current cipher */
@@ -386,6 +393,8 @@ SSL *SSL_new(SSL_CTX *ctx)
     s->psk_server_callback = ctx->psk_server_callback;
 #endif
 
+    s->job = NULL;
+
     return (s);
  err:
     SSL_free(s);
@@ -914,13 +923,55 @@ int SSL_check_private_key(const SSL *ssl)
                                    ssl->cert->key->privatekey));
 }
 
+int SSL_waiting_for_async(SSL *s)
+{
+    if(s->job) {
+        return ASYNC_job_is_waiting(s->job);
+    }
+    return 0;
+}
+
+static int ssl_accept_intern(void *vargs)
+{
+    struct ssl_async_args *args;
+    SSL *s;
+
+    args = (struct ssl_async_args *)vargs;
+    s = args->s;
+
+    return s->method->ssl_accept(s);
+}
+
 int SSL_accept(SSL *s)
 {
+    int ret;
+    struct ssl_async_args args;
+
     if (s->handshake_func == 0)
         /* Not properly initialized yet */
         SSL_set_accept_state(s);
 
-    return (s->method->ssl_accept(s));
+    args.s = s;
+
+    if((s->mode & SSL_MODE_ASYNC) && !ASYNC_in_job()) {
+        switch(ASYNC_start_job(&s->job, &ret, ssl_accept_intern, &args,
+            sizeof(struct ssl_async_args))) {
+        case ASYNC_ERR:
+            SSLerr(SSL_F_SSL_ACCEPT, SSL_R_FAILED_TO_INIT_ASYNC);
+            return -1;
+        case ASYNC_PAUSE:
+            return -1;
+        case ASYNC_FINISH:
+            s->job = NULL;
+            return ret;
+        default:
+            SSLerr(SSL_F_SSL_ACCEPT, ERR_R_INTERNAL_ERROR);
+            /* Shouldn't happen */
+            return -1;
+        }
+    } else {
+        return s->method->ssl_accept(s);
+    }
 }
 
 int SSL_connect(SSL *s)
@@ -937,8 +988,27 @@ long SSL_get_default_timeout(const SSL *s)
     return (s->method->get_timeout());
 }
 
+
+static int ssl_read_intern(void *vargs)
+{
+    struct ssl_async_args *args;
+    SSL *s;
+    void *buf;
+    int num;
+
+    args = (struct ssl_async_args *)vargs;
+    s = args->s;
+    buf = args->buf;
+    num = args->num;
+
+    return s->method->ssl_read(s, buf, num);
+}
+
 int SSL_read(SSL *s, void *buf, int num)
 {
+    int ret;
+    struct ssl_async_args args;
+
     if (s->handshake_func == 0) {
         SSLerr(SSL_F_SSL_READ, SSL_R_UNINITIALIZED);
         return -1;
@@ -948,7 +1018,33 @@ int SSL_read(SSL *s, void *buf, int num)
         s->rwstate = SSL_NOTHING;
         return (0);
     }
-    return (s->method->ssl_read(s, buf, num));
+
+    args.s = s;
+    args.buf = buf;
+    args.num = num;
+
+    if((s->mode & SSL_MODE_ASYNC) && !ASYNC_in_job()) {
+        switch(ASYNC_start_job(&s->job, &ret, ssl_read_intern, &args,
+            sizeof(struct ssl_async_args))) {
+        case ASYNC_ERR:
+            s->rwstate = SSL_NOTHING;
+            SSLerr(SSL_F_SSL_READ, SSL_R_FAILED_TO_INIT_ASYNC);
+            return -1;
+        case ASYNC_PAUSE:
+            s->rwstate = SSL_ASYNC_PAUSED;
+            return -1;
+        case ASYNC_FINISH:
+            s->job = NULL;
+            return ret;
+        default:
+            s->rwstate = SSL_NOTHING;
+            SSLerr(SSL_F_SSL_READ, ERR_R_INTERNAL_ERROR);
+            /* Shouldn't happen */
+            return -1;
+        }
+    } else {
+        return s->method->ssl_read(s, buf, num);
+    }
 }
 
 int SSL_peek(SSL *s, void *buf, int num)
@@ -964,8 +1060,27 @@ int SSL_peek(SSL *s, void *buf, int num)
     return (s->method->ssl_peek(s, buf, num));
 }
 
+static int ssl_write_intern(void *vargs)
+{
+    struct ssl_async_args *args;
+    SSL *s;
+    const void *buf;
+    int num;
+
+    args = (struct ssl_async_args *)vargs;
+    s = args->s;
+    buf = args->buf;
+    num = args->num;
+
+    return s->method->ssl_write(s, buf, num);
+}
+
+
 int SSL_write(SSL *s, const void *buf, int num)
 {
+    int ret;
+    struct ssl_async_args args;
+
     if (s->handshake_func == 0) {
         SSLerr(SSL_F_SSL_WRITE, SSL_R_UNINITIALIZED);
         return -1;
@@ -976,7 +1091,33 @@ int SSL_write(SSL *s, const void *buf, int num)
         SSLerr(SSL_F_SSL_WRITE, SSL_R_PROTOCOL_IS_SHUTDOWN);
         return (-1);
     }
-    return (s->method->ssl_write(s, buf, num));
+
+    args.s = s;
+    args.buf = (void *) buf;
+    args.num = num;
+
+    if((s->mode & SSL_MODE_ASYNC) && !ASYNC_in_job()) {
+        switch(ASYNC_start_job(&s->job, &ret, ssl_write_intern, &args,
+            sizeof(struct ssl_async_args))) {
+        case ASYNC_ERR:
+            s->rwstate = SSL_NOTHING;
+            SSLerr(SSL_F_SSL_WRITE, SSL_R_FAILED_TO_INIT_ASYNC);
+            return -1;
+        case ASYNC_PAUSE:
+            s->rwstate = SSL_ASYNC_PAUSED;
+            return -1;
+        case ASYNC_FINISH:
+            s->job = NULL;
+            return ret;
+        default:
+            s->rwstate = SSL_NOTHING;
+            SSLerr(SSL_F_SSL_WRITE, ERR_R_INTERNAL_ERROR);
+            /* Shouldn't happen */
+            return -1;
+        }
+    } else {
+        return s->method->ssl_write(s, buf, num);
+    }
 }
 
 int SSL_shutdown(SSL *s)
@@ -2373,6 +2514,9 @@ int SSL_get_error(const SSL *s, int i)
     if ((i < 0) && SSL_want_x509_lookup(s)) {
         return (SSL_ERROR_WANT_X509_LOOKUP);
     }
+    if ((i < 0) && SSL_want_async(s)) {
+        return SSL_ERROR_WANT_ASYNC;
+    }
 
     if (i == 0) {
         if ((s->shutdown & SSL_RECEIVED_SHUTDOWN) &&
@@ -2432,8 +2576,6 @@ int ssl_undefined_void_function(void)
 
 int ssl_undefined_const_function(const SSL *s)
 {
-    SSLerr(SSL_F_SSL_UNDEFINED_CONST_FUNCTION,
-           ERR_R_SHOULD_NOT_HAVE_BEEN_CALLED);
     return (0);
 }
 

ssl/ssl_locl.h

@@ -163,6 +163,7 @@
 # endif
 # include <openssl/err.h>
 # include <openssl/ssl.h>
+# include <openssl/async.h>
 # include <openssl/symhacks.h>
 
 #include "record/record.h"
@@ -1199,6 +1200,9 @@ struct ssl_st {
 
     /* Default password callback user data. */
     void *default_passwd_callback_userdata;
+
+    /* Async Job info */
+    ASYNC_JOB *job;
 };
 
 

test/Makefile

@@ -539,7 +539,8 @@ gost2814789test.o: ../include/openssl/pkcs7.h ../include/openssl/safestack.h
 gost2814789test.o: ../include/openssl/sha.h ../include/openssl/stack.h
 gost2814789test.o: ../include/openssl/symhacks.h ../include/openssl/x509.h
 gost2814789test.o: ../include/openssl/x509_vfy.h gost2814789test.c
-heartbeat_test.o: ../e_os.h ../include/openssl/asn1.h ../include/openssl/bio.h
+heartbeat_test.o: ../e_os.h ../include/openssl/asn1.h
+heartbeat_test.o: ../include/openssl/async.h ../include/openssl/bio.h
 heartbeat_test.o: ../include/openssl/bn.h ../include/openssl/buffer.h
 heartbeat_test.o: ../include/openssl/comp.h ../include/openssl/crypto.h
 heartbeat_test.o: ../include/openssl/dsa.h ../include/openssl/dtls1.h
@@ -659,30 +660,30 @@ sha1test.o: ../include/openssl/objects.h ../include/openssl/opensslconf.h
 sha1test.o: ../include/openssl/opensslv.h ../include/openssl/ossl_typ.h
 sha1test.o: ../include/openssl/safestack.h ../include/openssl/sha.h
 sha1test.o: ../include/openssl/stack.h ../include/openssl/symhacks.h sha1test.c
-ssltest.o: ../e_os.h ../include/openssl/asn1.h ../include/openssl/bio.h
-ssltest.o: ../include/openssl/bn.h ../include/openssl/buffer.h
-ssltest.o: ../include/openssl/comp.h ../include/openssl/conf.h
-ssltest.o: ../include/openssl/crypto.h ../include/openssl/dh.h
-ssltest.o: ../include/openssl/dsa.h ../include/openssl/dtls1.h
-ssltest.o: ../include/openssl/e_os2.h ../include/openssl/ec.h
-ssltest.o: ../include/openssl/ecdh.h ../include/openssl/ecdsa.h
-ssltest.o: ../include/openssl/engine.h ../include/openssl/err.h
-ssltest.o: ../include/openssl/evp.h ../include/openssl/hmac.h
-ssltest.o: ../include/openssl/lhash.h ../include/openssl/obj_mac.h
-ssltest.o: ../include/openssl/objects.h ../include/openssl/opensslconf.h
-ssltest.o: ../include/openssl/opensslv.h ../include/openssl/ossl_typ.h
-ssltest.o: ../include/openssl/pem.h ../include/openssl/pem2.h
-ssltest.o: ../include/openssl/pkcs7.h ../include/openssl/pqueue.h
-ssltest.o: ../include/openssl/rand.h ../include/openssl/rsa.h
-ssltest.o: ../include/openssl/safestack.h ../include/openssl/sha.h
-ssltest.o: ../include/openssl/srp.h ../include/openssl/srtp.h
-ssltest.o: ../include/openssl/ssl.h ../include/openssl/ssl2.h
-ssltest.o: ../include/openssl/ssl3.h ../include/openssl/stack.h
-ssltest.o: ../include/openssl/symhacks.h ../include/openssl/tls1.h
-ssltest.o: ../include/openssl/x509.h ../include/openssl/x509_vfy.h
-ssltest.o: ../include/openssl/x509v3.h ../ssl/packet_locl.h
-ssltest.o: ../ssl/record/record.h ../ssl/ssl_locl.h ../ssl/statem/statem.h
-ssltest.o: ssltest.c
+ssltest.o: ../e_os.h ../include/openssl/asn1.h ../include/openssl/async.h
+ssltest.o: ../include/openssl/bio.h ../include/openssl/bn.h
+ssltest.o: ../include/openssl/buffer.h ../include/openssl/comp.h
+ssltest.o: ../include/openssl/conf.h ../include/openssl/crypto.h
+ssltest.o: ../include/openssl/dh.h ../include/openssl/dsa.h
+ssltest.o: ../include/openssl/dtls1.h ../include/openssl/e_os2.h
+ssltest.o: ../include/openssl/ec.h ../include/openssl/ecdh.h
+ssltest.o: ../include/openssl/ecdsa.h ../include/openssl/engine.h
+ssltest.o: ../include/openssl/err.h ../include/openssl/evp.h
+ssltest.o: ../include/openssl/hmac.h ../include/openssl/lhash.h
+ssltest.o: ../include/openssl/obj_mac.h ../include/openssl/objects.h
+ssltest.o: ../include/openssl/opensslconf.h ../include/openssl/opensslv.h
+ssltest.o: ../include/openssl/ossl_typ.h ../include/openssl/pem.h
+ssltest.o: ../include/openssl/pem2.h ../include/openssl/pkcs7.h
+ssltest.o: ../include/openssl/pqueue.h ../include/openssl/rand.h
+ssltest.o: ../include/openssl/rsa.h ../include/openssl/safestack.h
+ssltest.o: ../include/openssl/sha.h ../include/openssl/srp.h
+ssltest.o: ../include/openssl/srtp.h ../include/openssl/ssl.h
+ssltest.o: ../include/openssl/ssl2.h ../include/openssl/ssl3.h
+ssltest.o: ../include/openssl/stack.h ../include/openssl/symhacks.h
+ssltest.o: ../include/openssl/tls1.h ../include/openssl/x509.h
+ssltest.o: ../include/openssl/x509_vfy.h ../include/openssl/x509v3.h
+ssltest.o: ../ssl/packet_locl.h ../ssl/record/record.h ../ssl/ssl_locl.h
+ssltest.o: ../ssl/statem/statem.h ssltest.c
 testutil.o: ../e_os.h ../include/openssl/e_os2.h
 testutil.o: ../include/openssl/opensslconf.h testutil.c testutil.h
 v3nametest.o: ../e_os.h ../include/openssl/asn1.h ../include/openssl/bio.h