Add TFO support to socket BIO and s_client/s_server

Supports Linux, MacOS and FreeBSD
Disabled by default, enabled via `enabled-tfo`
Some tests

Reviewed-by: Matt Caswell <matt@openssl.org>
Reviewed-by: Tomas Mraz <tomas@openssl.org>
Reviewed-by: Tim Hudson <tjh@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/8692)

.github/workflows/ci.yml

@@ -216,6 +216,20 @@ jobs:
     - name: make test
       run: make test HARNESS_JOBS=${HARNESS_JOBS:-4}
 
+  enable-tfo:
+    strategy:
+      matrix:
+        os: [ ubuntu-latest, macos-latest ]
+    runs-on: ${{matrix.os}}
+    steps:
+    - uses: actions/checkout@v2
+    - name: config
+      run: CC=gcc ./config --banner=Configured enable-tfo --strict-warnings && perl configdata.pm --dump
+    - name: make
+      run: make -s -j4
+    - name: make test
+      run: make test HARNESS_JOBS=${HARNESS_JOBS:-4}
+
   buildtest:
     runs-on: ubuntu-latest
     steps:

.github/workflows/run-checker-daily.yml

@@ -104,6 +104,7 @@ jobs:
           no-sm2,
           no-sm3,
           no-sm4,
+          no-sock,
           no-sse2,
           no-ssl,
           no-ssl3,
@@ -111,6 +112,7 @@ jobs:
           no-ssl-trace,
           no-static-engine no-shared,
           no-stdio,
+          enable-tfo,
           no-tls1,
           no-tls1_1,
           no-tls1_1-method,

CHANGES.md

@@ -24,6 +24,11 @@ OpenSSL 3.1
 
 ### Changes between 3.0 and 3.1 [xx XXX xxxx]
 
+ * Add support for TCP Fast Open (RFC7413) to macOS, Linux, and FreeBSD where
+   supported and enabled.
+
+   *Todd Short*
+
  * Add ciphersuites based on DHE_PSK (RFC 4279) and ECDHE_PSK (RFC 5489)
    to the list of ciphersuites providing Perfect Forward Secrecy as
    required by SECLEVEL >= 3.

Configure

@@ -492,6 +492,7 @@ my @disablables = (
     "static-engine",
     "stdio",
     "tests",
+    "tfo",
     "threads",
     "tls",
     "trace",
@@ -551,6 +552,7 @@ our %disabled = ( # "what"         => "comment"
                   "sctp"                => "default",
                   "ssl3"                => "default",
                   "ssl3-method"         => "default",
+                  "tfo"                 => "default",
                   "trace"               => "default",
                   "ubsan"               => "default",
                   "unit-test"           => "default",
@@ -576,6 +578,7 @@ my @disable_cascades = (
                              "seed", "siphash", "siv",
                              "sm3", "sm4", "srp",
                              "srtp", "ssl3-method", "ssl-trace",
+                             "tfo",
                              "ts", "ui-console", "whirlpool",
                              "fips-securitychecks" ],
     sub { $config{processor} eq "386" }
@@ -586,7 +589,7 @@ my @disable_cascades = (
     "des"               => [ "mdc2" ],
     "ec"                => [ "ec2m", "ecdsa", "ecdh", "sm2", "gost" ],
     "dgram"             => [ "dtls", "sctp" ],
-    "sock"              => [ "dgram" ],
+    "sock"              => [ "dgram", "tfo" ],
     "dtls"              => [ @dtls ],
     sub { 0 == scalar grep { !$disabled{$_} } @dtls }
                         => [ "dtls" ],

INSTALL.md

@@ -878,6 +878,10 @@ tests also use the command line applications, the tests will also be skipped.
 
 Don't build test programs or run any tests.
 
+### enable-tfo
+
+Build with support for TCP Fast Open (RFC7413). Supported on Linux, macOS and FreeBSD.
+
 ### no-threads
 
 Don't build with support for multi-threaded applications.

NEWS.md

@@ -23,6 +23,8 @@ OpenSSL 3.1
 
   * Subject or issuer names in X.509 objects are now displayed as UTF-8 strings
     by default.
+  * TCP Fast Open (RFC7413) support is available on Linux, macOS, and FreeBSD
+    where enabled and supported.
 
 OpenSSL 3.0
 -----------

apps/include/s_apps.h

@@ -23,8 +23,8 @@ void get_sock_info_address(int asock, char **hostname, char **service);
 int report_server_accept(BIO *out, int asock, int with_address, int with_pid);
 int do_server(int *accept_sock, const char *host, const char *port,
               int family, int type, int protocol, do_server_cb cb,
-              unsigned char *context, int naccept, BIO *bio_s_out);
+              unsigned char *context, int naccept, BIO *bio_s_out,
-
+              int tfo);
 int verify_callback(int ok, X509_STORE_CTX *ctx);
 
 int set_cert_stuff(SSL_CTX *ctx, char *cert_file, char *key_file);
@@ -36,7 +36,8 @@ int ssl_print_groups(BIO *out, SSL *s, int noshared);
 int ssl_print_tmp_key(BIO *out, SSL *s);
 int init_client(int *sock, const char *host, const char *port,
                 const char *bindhost, const char *bindport,
-                int family, int type, int protocol);
+                int family, int type, int protocol, int tfo,
+                BIO_ADDR **ba_ret);
 int should_retry(int i);
 void do_ssl_shutdown(SSL *ssl);
 

apps/lib/s_socket.c

@@ -64,6 +64,8 @@ BIO_ADDR *ourpeer = NULL;
  *  AF_UNSPEC
  * @type: socket type, must be SOCK_STREAM or SOCK_DGRAM
  * @protocol: socket protocol, e.g. IPPROTO_TCP or IPPROTO_UDP (or 0 for any)
+ * @tfo: flag to enable TCP Fast Open
+ * @ba_ret: BIO_ADDR that was connected to for TFO, to be freed by caller
  *
  * This will create a socket and use it to connect to a host:port, or if
  * family == AF_UNIX, to the path found in host.
@@ -76,7 +78,8 @@ BIO_ADDR *ourpeer = NULL;
  */
 int init_client(int *sock, const char *host, const char *port,
                 const char *bindhost, const char *bindport,
-                int family, int type, int protocol)
+                int family, int type, int protocol, int tfo,
+                BIO_ADDR **ba_ret)
 {
     BIO_ADDRINFO *res = NULL;
     BIO_ADDRINFO *bindaddr = NULL;
@@ -84,6 +87,10 @@ int init_client(int *sock, const char *host, const char *port,
     const BIO_ADDRINFO *bi = NULL;
     int found = 0;
     int ret;
+    int options = 0;
+
+    if (tfo && ba_ret != NULL)
+        *ba_ret = NULL;
 
     if (BIO_sock_init() != 1)
         return 0;
@@ -160,14 +167,22 @@ int init_client(int *sock, const char *host, const char *port,
             BIO_free(tmpbio);
         }
 #endif
+        if (BIO_ADDRINFO_protocol(ai) == IPPROTO_TCP) {
+            options |= BIO_SOCK_NODELAY;
+            if (tfo)
+                options |= BIO_SOCK_TFO;
+        }
 
-        if (!BIO_connect(*sock, BIO_ADDRINFO_address(ai),
+        if (!BIO_connect(*sock, BIO_ADDRINFO_address(ai), options)) {
-                         BIO_ADDRINFO_protocol(ai) == IPPROTO_TCP ? BIO_SOCK_NODELAY : 0)) {
             BIO_closesocket(*sock);
             *sock = INVALID_SOCKET;
             continue;
         }
 
+        /* Save the address */
+        if (tfo && ba_ret != NULL)
+            *ba_ret = BIO_ADDR_dup(BIO_ADDRINFO_address(ai));
+
         /* Success, don't try any more addresses */
         break;
     }
@@ -278,7 +293,8 @@ int report_server_accept(BIO *out, int asock, int with_address, int with_pid)
  */
 int do_server(int *accept_sock, const char *host, const char *port,
               int family, int type, int protocol, do_server_cb cb,
-              unsigned char *context, int naccept, BIO *bio_s_out)
+              unsigned char *context, int naccept, BIO *bio_s_out,
+              int tfo)
 {
     int asock = 0;
     int sock;
@@ -312,6 +328,8 @@ int do_server(int *accept_sock, const char *host, const char *port,
     sock_protocol = BIO_ADDRINFO_protocol(res);
     sock_address = BIO_ADDRINFO_address(res);
     next = BIO_ADDRINFO_next(res);
+    if (tfo && sock_type == SOCK_STREAM)
+        sock_options |= BIO_SOCK_TFO;
 #ifdef AF_INET6
     if (sock_family == AF_INET6)
         sock_options |= BIO_SOCK_V6_ONLY;

apps/s_client.c

@@ -456,6 +456,7 @@ typedef enum OPTION_choice {
     OPT_USE_SRTP, OPT_KEYMATEXPORT, OPT_KEYMATEXPORTLEN, OPT_PROTOHOST,
     OPT_MAXFRAGLEN, OPT_MAX_SEND_FRAG, OPT_SPLIT_SEND_FRAG, OPT_MAX_PIPELINES,
     OPT_READ_BUF, OPT_KEYLOG_FILE, OPT_EARLY_DATA, OPT_REQCAFILE,
+    OPT_TFO,
     OPT_V_ENUM,
     OPT_X_ENUM,
     OPT_S_ENUM, OPT_IGNORE_UNEXPECTED_EOF,
@@ -539,6 +540,9 @@ const OPTIONS s_client_options[] = {
      "Do not load certificates from the default certificates store"},
     {"requestCAfile", OPT_REQCAFILE, '<',
       "PEM format file of CA names to send to the server"},
+#if defined(TCP_FASTOPEN) && !defined(OPENSSL_NO_TFO)
+    {"tfo", OPT_TFO, '-', "Connect using TCP Fast Open"},
+#endif
     {"dane_tlsa_domain", OPT_DANE_TLSA_DOMAIN, 's', "DANE TLSA base domain"},
     {"dane_tlsa_rrdata", OPT_DANE_TLSA_RRDATA, 's',
      "DANE TLSA rrdata presentation form"},
@@ -899,6 +903,8 @@ int s_client_main(int argc, char **argv)
 #ifndef OPENSSL_NO_KTLS
     int enable_ktls = 0;
 #endif
+    int tfo = 0;
+    BIO_ADDR *tfo_addr = NULL;
 
     FD_ZERO(&readfds);
     FD_ZERO(&writefds);
@@ -1413,6 +1419,9 @@ int s_client_main(int argc, char **argv)
             if (!opt_pair(opt_arg(), services, &starttls_proto))
                 goto end;
             break;
+        case OPT_TFO:
+            tfo = 1;
+            break;
         case OPT_SERVERNAME:
             servername = opt_arg();
             break;
@@ -2035,10 +2044,18 @@ int s_client_main(int argc, char **argv)
                    "-dane_tlsa_domain option.\n", prog);
         goto end;
     }
+#ifndef OPENSSL_NO_DTLS
+    if (isdtls && tfo) {
+        BIO_printf(bio_err, "%s: DTLS does not support the -tfo option\n", prog);
+        goto end;
+    }
+#endif
 
+    if (tfo)
+        BIO_printf(bio_c_out, "Connecting via TFO\n");
  re_start:
     if (init_client(&sock, host, port, bindhost, bindport, socket_family,
-                    socket_type, protocol) == 0) {
+                    socket_type, protocol, tfo, &tfo_addr) == 0) {
         BIO_printf(bio_err, "connect:errno=%d\n", get_last_socket_error());
         BIO_closesocket(sock);
         goto end;
@@ -2120,6 +2137,12 @@ int s_client_main(int argc, char **argv)
         goto end;
     }
 
+    /* Now that we're using a BIO... */
+    if (tfo_addr != NULL)
+        (void)BIO_set_conn_address(sbio, tfo_addr);
+    if (tfo)
+        (void)BIO_set_tfo(sbio, 1);
+
     if (nbio_test) {
         BIO *test;
 
@@ -2909,9 +2932,12 @@ int s_client_main(int argc, char **argv)
 
             case SSL_ERROR_SYSCALL:
                 if ((k != 0) || (cbuf_len != 0)) {
-                    BIO_printf(bio_err, "write:errno=%d\n",
+                    int sockerr = get_last_socket_error();
-                               get_last_socket_error());
+
-                    goto shut;
+                    if (!tfo || sockerr != EISCONN) {
+                        BIO_printf(bio_err, "write:errno=%d\n", sockerr);
+                        goto shut;
+                    }
                 } else {
                     read_tty = 1;
                     write_ssl = 0;
@@ -3131,6 +3157,7 @@ int s_client_main(int argc, char **argv)
     OPENSSL_free(srp_arg.srppassin);
 #endif
     OPENSSL_free(sname_alloc);
+    BIO_ADDR_free(tfo_addr);
     OPENSSL_free(connectstr);
     OPENSSL_free(bindstr);
     OPENSSL_free(bindhost);

apps/s_server.c

@@ -717,6 +717,7 @@ typedef enum OPTION_choice {
     OPT_KEYLOG_FILE, OPT_MAX_EARLY, OPT_RECV_MAX_EARLY, OPT_EARLY_DATA,
     OPT_S_NUM_TICKETS, OPT_ANTI_REPLAY, OPT_NO_ANTI_REPLAY, OPT_SCTP_LABEL_BUG,
     OPT_HTTP_SERVER_BINMODE, OPT_NOCANAMES, OPT_IGNORE_UNEXPECTED_EOF, OPT_KTLS,
+    OPT_TFO,
     OPT_R_ENUM,
     OPT_S_ENUM,
     OPT_V_ENUM,
@@ -747,6 +748,9 @@ const OPTIONS s_server_options[] = {
 #endif
     {"4", OPT_4, '-', "Use IPv4 only"},
     {"6", OPT_6, '-', "Use IPv6 only"},
+#if defined(TCP_FASTOPEN) && !defined(OPENSSL_NO_TFO)
+    {"tfo", OPT_TFO, '-', "Listen for TCP Fast Open connections"},
+#endif
 
     OPT_SECTION("Identity"),
     {"context", OPT_CONTEXT, 's', "Set session ID context"},
@@ -1057,6 +1061,7 @@ int s_server_main(int argc, char *argv[])
 #ifndef OPENSSL_NO_KTLS
     int enable_ktls = 0;
 #endif
+    int tfo = 0;
 
     /* Init of few remaining global variables */
     local_argc = argc;
@@ -1649,6 +1654,9 @@ int s_server_main(int argc, char *argv[])
         case OPT_IGNORE_UNEXPECTED_EOF:
             ignore_unexpected_eof = 1;
             break;
+        case OPT_TFO:
+            tfo = 1;
+            break;
         }
     }
 
@@ -1677,6 +1685,11 @@ int s_server_main(int argc, char *argv[])
     }
 #endif
 
+    if (tfo && socket_type != SOCK_STREAM) {
+        BIO_printf(bio_err, "Can only use -tfo with TLS\n");
+        goto end;
+    }
+
     if (stateless && socket_type != SOCK_STREAM) {
         BIO_printf(bio_err, "Can only use --stateless with TLS\n");
         goto end;
@@ -2240,8 +2253,10 @@ int s_server_main(int argc, char *argv[])
         && unlink_unix_path)
         unlink(host);
 #endif
+    if (tfo)
+        BIO_printf(bio_s_out, "Listening for TFO\n");
     do_server(&accept_socket, host, port, socket_family, socket_type, protocol,
-              server_cb, context, naccept, bio_s_out);
+              server_cb, context, naccept, bio_s_out, tfo);
     print_stats(bio_s_out, ctx);
     ret = 0;
  end:

crypto/bio/bio_addr.c

@@ -67,6 +67,18 @@ void BIO_ADDR_free(BIO_ADDR *ap)
     OPENSSL_free(ap);
 }
 
+BIO_ADDR *BIO_ADDR_dup(const BIO_ADDR *ap)
+{
+    BIO_ADDR *ret = NULL;
+
+    if (ap != NULL) {
+        ret = BIO_ADDR_new();
+        if (ret != NULL)
+            memcpy(ret, ap, sizeof(BIO_ADDR));
+    }
+    return ret;
+}
+
 void BIO_ADDR_clear(BIO_ADDR *ap)
 {
     memset(ap, 0, sizeof(*ap));

crypto/bio/bio_err.c

@@ -46,6 +46,9 @@ static const ERR_STRING_DATA BIO_str_reasons[] = {
     "no hostname or service specified"},
     {ERR_PACK(ERR_LIB_BIO, 0, BIO_R_NO_PORT_DEFINED), "no port defined"},
     {ERR_PACK(ERR_LIB_BIO, 0, BIO_R_NO_SUCH_FILE), "no such file"},
+    {ERR_PACK(ERR_LIB_BIO, 0, BIO_R_TFO_DISABLED), "tfo disabled"},
+    {ERR_PACK(ERR_LIB_BIO, 0, BIO_R_TFO_NO_KERNEL_SUPPORT),
+    "tfo no kernel support"},
     {ERR_PACK(ERR_LIB_BIO, 0, BIO_R_TRANSFER_ERROR), "transfer error"},
     {ERR_PACK(ERR_LIB_BIO, 0, BIO_R_TRANSFER_TIMEOUT), "transfer timeout"},
     {ERR_PACK(ERR_LIB_BIO, 0, BIO_R_UNABLE_TO_BIND_SOCKET),
@@ -59,6 +62,7 @@ static const ERR_STRING_DATA BIO_str_reasons[] = {
     {ERR_PACK(ERR_LIB_BIO, 0, BIO_R_UNABLE_TO_NODELAY), "unable to nodelay"},
     {ERR_PACK(ERR_LIB_BIO, 0, BIO_R_UNABLE_TO_REUSEADDR),
     "unable to reuseaddr"},
+    {ERR_PACK(ERR_LIB_BIO, 0, BIO_R_UNABLE_TO_TFO), "unable to tfo"},
     {ERR_PACK(ERR_LIB_BIO, 0, BIO_R_UNAVAILABLE_IP_FAMILY),
     "unavailable ip family"},
     {ERR_PACK(ERR_LIB_BIO, 0, BIO_R_UNINITIALIZED), "uninitialized"},

crypto/bio/bio_sock2.c

@@ -13,6 +13,7 @@
 
 #include "bio_local.h"
 #include "internal/ktls.h"
+#include "internal/bio_tfo.h"
 
 #include <openssl/err.h>
 
@@ -79,6 +80,7 @@ int BIO_socket(int domain, int socktype, int protocol, int options)
  * - BIO_SOCK_KEEPALIVE: enable regularly sending keep-alive messages.
  * - BIO_SOCK_NONBLOCK: Make the socket non-blocking.
  * - BIO_SOCK_NODELAY: don't delay small messages.
+ * - BIO_SOCK_TFO: use TCP Fast Open
  *
  * options holds BIO socket options that can be used
  * You should call this for every address returned by BIO_lookup
@@ -118,6 +120,68 @@ int BIO_connect(int sock, const BIO_ADDR *addr, int options)
             return 0;
         }
     }
+    if (options & BIO_SOCK_TFO) {
+# if defined(OSSL_TFO_CLIENT_FLAG)
+#  if defined(OSSL_TFO_SYSCTL_CLIENT)
+        int enabled = 0;
+        size_t enabledlen = sizeof(enabled);
+
+        /* Later FreeBSD */
+        if (sysctlbyname(OSSL_TFO_SYSCTL_CLIENT, &enabled, &enabledlen, NULL, 0) < 0) {
+            ERR_raise(ERR_LIB_BIO, BIO_R_TFO_NO_KERNEL_SUPPORT);
+            return 0;
+        }
+        /* Need to check for client flag */
+        if (!(enabled & OSSL_TFO_CLIENT_FLAG)) {
+            ERR_raise(ERR_LIB_BIO, BIO_R_TFO_DISABLED);
+            return 0;
+        }
+#  elif defined(OSSL_TFO_SYSCTL)
+        int enabled = 0;
+        size_t enabledlen = sizeof(enabled);
+
+        /* macOS */
+        if (sysctlbyname(OSSL_TFO_SYSCTL, &enabled, &enabledlen, NULL, 0) < 0) {
+            ERR_raise(ERR_LIB_BIO, BIO_R_TFO_NO_KERNEL_SUPPORT);
+            return 0;
+        }
+        /* Need to check for client flag */
+        if (!(enabled & OSSL_TFO_CLIENT_FLAG)) {
+            ERR_raise(ERR_LIB_BIO, BIO_R_TFO_DISABLED);
+            return 0;
+        }
+#  endif
+# endif
+# if defined(OSSL_TFO_CONNECTX)
+        sa_endpoints_t sae;
+
+        memset(&sae, 0, sizeof(sae));
+        sae.sae_dstaddr = BIO_ADDR_sockaddr(addr);
+        sae.sae_dstaddrlen = BIO_ADDR_sockaddr_size(addr);
+        if (connectx(sock, &sae, SAE_ASSOCID_ANY,
+                     CONNECT_DATA_IDEMPOTENT | CONNECT_RESUME_ON_READ_WRITE,
+                     NULL, 0, NULL, NULL) == -1) {
+            if (!BIO_sock_should_retry(-1)) {
+                ERR_raise_data(ERR_LIB_SYS, get_last_socket_error(),
+                               "calling connectx()");
+                ERR_raise(ERR_LIB_BIO, BIO_R_CONNECT_ERROR);
+            }
+            return 0;
+        }
+# endif
+# if defined(OSSL_TFO_CLIENT_SOCKOPT)
+        if (setsockopt(sock, IPPROTO_TCP, OSSL_TFO_CLIENT_SOCKOPT,
+                       (const void *)&on, sizeof(on)) != 0) {
+            ERR_raise_data(ERR_LIB_SYS, get_last_socket_error(),
+                           "calling setsockopt()");
+            ERR_raise(ERR_LIB_BIO, BIO_R_UNABLE_TO_TFO);
+            return 0;
+        }
+# endif
+# if defined(OSSL_TFO_DO_NOT_CONNECT)
+        return 1;
+# endif
+    }
 
     if (connect(sock, BIO_ADDR_sockaddr(addr),
                 BIO_ADDR_sockaddr_size(addr)) == -1) {
@@ -201,6 +265,7 @@ int BIO_bind(int sock, const BIO_ADDR *addr, int options)
  *   for a recently closed port.
  * - BIO_SOCK_V6_ONLY: When creating an IPv6 socket, make it listen only
  *   for IPv6 addresses and not IPv4 addresses mapped to IPv6.
+ * - BIO_SOCK_TFO: accept TCP fast open (set TCP_FASTOPEN)
  *
  * It's recommended that you set up both an IPv6 and IPv4 listen socket, and
  * then check both for new clients that connect to it.  You want to set up
@@ -292,6 +357,54 @@ int BIO_listen(int sock, const BIO_ADDR *addr, int options)
         return 0;
     }
 
+# if defined(OSSL_TFO_SERVER_SOCKOPT)
+    /*
+     * Must do it explicitly after listen() for macOS, still
+     * works fine on other OS's
+     */
+    if ((options & BIO_SOCK_TFO) && socktype != SOCK_DGRAM) {
+        int q = OSSL_TFO_SERVER_SOCKOPT_VALUE;
+#  if defined(OSSL_TFO_CLIENT_FLAG)
+#   if defined(OSSL_TFO_SYSCTL_SERVER)
+        int enabled = 0;
+        size_t enabledlen = sizeof(enabled);
+
+        /* Later FreeBSD */
+        if (sysctlbyname(OSSL_TFO_SYSCTL_SERVER, &enabled, &enabledlen, NULL, 0) < 0) {
+            ERR_raise(ERR_LIB_BIO, BIO_R_TFO_NO_KERNEL_SUPPORT);
+            return 0;
+        }
+        /* Need to check for server flag */
+        if (!(enabled & OSSL_TFO_SERVER_FLAG)) {
+            ERR_raise(ERR_LIB_BIO, BIO_R_TFO_DISABLED);
+            return 0;
+        }
+#   elif defined(OSSL_TFO_SYSCTL)
+        int enabled = 0;
+        size_t enabledlen = sizeof(enabled);
+
+        /* Early FreeBSD, macOS */
+        if (sysctlbyname(OSSL_TFO_SYSCTL, &enabled, &enabledlen, NULL, 0) < 0) {
+            ERR_raise(ERR_LIB_BIO, BIO_R_TFO_NO_KERNEL_SUPPORT);
+            return 0;
+        }
+        /* Need to check for server flag */
+        if (!(enabled & OSSL_TFO_SERVER_FLAG)) {
+            ERR_raise(ERR_LIB_BIO, BIO_R_TFO_DISABLED);
+            return 0;
+        }
+#   endif
+#  endif
+        if (setsockopt(sock, IPPROTO_TCP, OSSL_TFO_SERVER_SOCKOPT,
+                       (void *)&q, sizeof(q)) < 0) {
+            ERR_raise_data(ERR_LIB_SYS, get_last_socket_error(),
+                           "calling setsockopt()");
+            ERR_raise(ERR_LIB_BIO, BIO_R_UNABLE_TO_TFO);
+            return 0;
+        }
+    }
+# endif
+
     return 1;
 }
 

crypto/bio/bss_acpt.c

@@ -452,10 +452,14 @@ static long acpt_ctrl(BIO *b, int cmd, long num, void *ptr)
                 data->bio_chain = (BIO *)ptr;
             } else if (num == 4) {
                 data->accept_family = *(int *)ptr;
+            } else if (num == 5) {
+                data->bind_mode |= BIO_SOCK_TFO;
             }
         } else {
             if (num == 2) {
                 data->bind_mode &= ~BIO_SOCK_NONBLOCK;
+            } else if (num == 5) {
+                data->bind_mode &= ~BIO_SOCK_TFO;
             }
         }
         break;

crypto/bio/bss_conn.c

@@ -11,6 +11,7 @@
 #include <errno.h>
 
 #include "bio_local.h"
+#include "internal/bio_tfo.h"
 #include "internal/ktls.h"
 
 #ifndef OPENSSL_NO_SOCK
@@ -24,6 +25,7 @@ typedef struct bio_connect_st {
 # ifndef OPENSSL_NO_KTLS
     unsigned char record_type;
 # endif
+    int tfo_first;
 
     BIO_ADDRINFO *addr_first;
     const BIO_ADDRINFO *addr_iter;
@@ -360,6 +362,15 @@ static int conn_write(BIO *b, const char *in, int inl)
             BIO_clear_ktls_ctrl_msg_flag(b);
         }
     } else
+# endif
+# if defined(OSSL_TFO_SENDTO)
+    if (data->tfo_first) {
+        int peerlen = BIO_ADDRINFO_sockaddr_size(data->addr_iter);
+
+        ret = sendto(b->num, in, inl, OSSL_TFO_SENDTO,
+                     BIO_ADDRINFO_sockaddr(data->addr_iter), peerlen);
+        data->tfo_first = 0;
+    } else
 # endif
         ret = writesocket(b->num, in, inl);
     BIO_clear_retry_flags(b);
@@ -425,6 +436,8 @@ static long conn_ctrl(BIO *b, int cmd, long num, void *ptr)
                     ret = -1;
                     break;
                 }
+            } else if (num == 4) {
+                ret = data->connect_mode;
             } else {
                 ret = 0;
             }
@@ -485,8 +498,23 @@ static long conn_ctrl(BIO *b, int cmd, long num, void *ptr)
         else
             data->connect_mode &= ~BIO_SOCK_NONBLOCK;
         break;
+#if defined(TCP_FASTOPEN) && !defined(OPENSSL_NO_TFO)
+    case BIO_C_SET_TFO:
+        if (num != 0) {
+            data->connect_mode |= BIO_SOCK_TFO;
+            data->tfo_first = 1;
+        } else {
+            data->connect_mode &= ~BIO_SOCK_TFO;
+            data->tfo_first = 0;
+        }
+        break;
+#endif
     case BIO_C_SET_CONNECT_MODE:
         data->connect_mode = (int)num;
+        if (num & BIO_SOCK_TFO)
+            data->tfo_first = 1;
+        else
+            data->tfo_first = 0;
         break;
     case BIO_C_GET_FD:
         if (b->init) {

crypto/bio/bss_sock.c

@@ -10,6 +10,7 @@
 #include <stdio.h>
 #include <errno.h>
 #include "bio_local.h"
+#include "internal/bio_tfo.h"
 #include "internal/cryptlib.h"
 #include "internal/ktls.h"
 
@@ -27,6 +28,14 @@
 #  define sock_puts  SockPuts
 # endif
 
+struct bss_sock_st {
+    BIO_ADDR tfo_peer;
+    int tfo_first;
+#ifndef OPENSSL_NO_KTLS
+    unsigned char ktls_record_type;
+#endif
+};
+
 static int sock_write(BIO *h, const char *buf, int num);
 static int sock_read(BIO *h, char *buf, int size);
 static int sock_puts(BIO *h, const char *str);
@@ -81,8 +90,10 @@ static int sock_new(BIO *bi)
 {
     bi->init = 0;
     bi->num = 0;
-    bi->ptr = NULL;
     bi->flags = 0;
+    bi->ptr = OPENSSL_zalloc(sizeof(struct bss_sock_st));
+    if (bi->ptr == NULL)
+        return 0;
     return 1;
 }
 
@@ -97,6 +108,8 @@ static int sock_free(BIO *a)
         a->init = 0;
         a->flags = 0;
     }
+    OPENSSL_free(a->ptr);
+    a->ptr = NULL;
     return 1;
 }
 
@@ -126,17 +139,30 @@ static int sock_read(BIO *b, char *out, int outl)
 static int sock_write(BIO *b, const char *in, int inl)
 {
     int ret = 0;
+# if !defined(OPENSSL_NO_KTLS) || defined(OSSL_TFO_SENDTO)
+    struct bss_sock_st *data = (struct bss_sock_st *)b->ptr;
+# endif
 
     clear_socket_error();
 # ifndef OPENSSL_NO_KTLS
     if (BIO_should_ktls_ctrl_msg_flag(b)) {
-        unsigned char record_type = (intptr_t)b->ptr;
+        unsigned char record_type = data->ktls_record_type;
         ret = ktls_send_ctrl_message(b->num, record_type, in, inl);
         if (ret >= 0) {
             ret = inl;
             BIO_clear_ktls_ctrl_msg_flag(b);
         }
     } else
+# endif
+# if defined(OSSL_TFO_SENDTO)
+    if (data->tfo_first) {
+        struct bss_sock_st *data = (struct bss_sock_st *)b->ptr;
+        socklen_t peerlen = BIO_ADDR_sockaddr_size(&data->tfo_peer);
+
+        ret = sendto(b->num, in, inl, OSSL_TFO_SENDTO,
+                     BIO_ADDR_sockaddr(&data->tfo_peer), peerlen);
+        data->tfo_first = 0;
+    } else
 # endif
         ret = writesocket(b->num, in, inl);
     BIO_clear_retry_flags(b);
@@ -151,16 +177,24 @@ static long sock_ctrl(BIO *b, int cmd, long num, void *ptr)
 {
     long ret = 1;
     int *ip;
+    struct bss_sock_st *data = (struct bss_sock_st *)b->ptr;
 # ifndef OPENSSL_NO_KTLS
     ktls_crypto_info_t *crypto_info;
 # endif
 
     switch (cmd) {
     case BIO_C_SET_FD:
-        sock_free(b);
+        /* minimal sock_free() */
+        if (b->shutdown) {
+            if (b->init)
+                BIO_closesocket(b->num);
+            b->flags = 0;
+        }
         b->num = *((int *)ptr);
         b->shutdown = (int)num;
         b->init = 1;
+        data->tfo_first = 0;
+        memset(&data->tfo_peer, 0, sizeof(data->tfo_peer));
         break;
     case BIO_C_GET_FD:
         if (b->init) {
@@ -194,7 +228,7 @@ static long sock_ctrl(BIO *b, int cmd, long num, void *ptr)
         return BIO_should_ktls_flag(b, 0) != 0;
     case BIO_CTRL_SET_KTLS_TX_SEND_CTRL_MSG:
         BIO_set_ktls_ctrl_msg_flag(b);
-        b->ptr = (void *)num;
+        data->ktls_record_type = (unsigned char)num;
         ret = 0;
         break;
     case BIO_CTRL_CLEAR_KTLS_TX_CTRL_MSG:
@@ -205,6 +239,25 @@ static long sock_ctrl(BIO *b, int cmd, long num, void *ptr)
     case BIO_CTRL_EOF:
         ret = (b->flags & BIO_FLAGS_IN_EOF) != 0;
         break;
+    case BIO_C_GET_CONNECT:
+        if (ptr != NULL && num == 2) {
+            const char **pptr = (const char **)ptr;
+
+            *pptr = (const char *)&data->tfo_peer;
+        } else {
+            ret = 0;
+        }
+        break;
+    case BIO_C_SET_CONNECT:
+        if (ptr != NULL && num == 2) {
+            ret = BIO_ADDR_make(&data->tfo_peer,
+                                BIO_ADDR_sockaddr((const BIO_ADDR *)ptr));
+            if (ret)
+                data->tfo_first = 1;
+        } else {
+            ret = 0;
+        }
+        break;
     default:
         ret = 0;
         break;

crypto/err/openssl.txt

@@ -150,6 +150,8 @@ BIO_R_NO_ACCEPT_ADDR_OR_SERVICE_SPECIFIED:143:\
 BIO_R_NO_HOSTNAME_OR_SERVICE_SPECIFIED:144:no hostname or service specified
 BIO_R_NO_PORT_DEFINED:113:no port defined
 BIO_R_NO_SUCH_FILE:128:no such file
+BIO_R_TFO_DISABLED:106:tfo disabled
+BIO_R_TFO_NO_KERNEL_SUPPORT:108:tfo no kernel support
 BIO_R_TRANSFER_ERROR:104:transfer error
 BIO_R_TRANSFER_TIMEOUT:105:transfer timeout
 BIO_R_UNABLE_TO_BIND_SOCKET:117:unable to bind socket
@@ -158,6 +160,7 @@ BIO_R_UNABLE_TO_KEEPALIVE:137:unable to keepalive
 BIO_R_UNABLE_TO_LISTEN_SOCKET:119:unable to listen socket
 BIO_R_UNABLE_TO_NODELAY:138:unable to nodelay
 BIO_R_UNABLE_TO_REUSEADDR:139:unable to reuseaddr
+BIO_R_UNABLE_TO_TFO:109:unable to tfo
 BIO_R_UNAVAILABLE_IP_FAMILY:145:unavailable ip family
 BIO_R_UNINITIALIZED:120:uninitialized
 BIO_R_UNKNOWN_INFO_TYPE:140:unknown info type

doc/man1/openssl-s_client.pod.in

@@ -119,6 +119,7 @@ B<openssl> B<s_client>
 [B<-srp_moregroups>]
 [B<-srp_strength> I<number>]
 [B<-ktls>]
+[B<-tfo>]
 {- $OpenSSL::safe::opt_name_synopsis -}
 {- $OpenSSL::safe::opt_version_synopsis -}
 {- $OpenSSL::safe::opt_x_synopsis -}
@@ -782,6 +783,10 @@ Enable Kernel TLS for sending and receiving.
 This option was introduced in OpenSSL 3.1.0.
 Kernel TLS is off by default as of OpenSSL 3.1.0.
 
+=item B<-tfo>
+
+Enable creation of connections via TCP fast open (RFC7413).
+
 {- $OpenSSL::safe::opt_version_item -}
 
 {- $OpenSSL::safe::opt_name_item -}
@@ -924,6 +929,8 @@ The B<-certform> option has become obsolete in OpenSSL 3.0.0 and has no effect.
 
 The B<-engine> option was deprecated in OpenSSL 3.0.
 
+The -tfo option was added in OpenSSL 3.1.
+
 =head1 COPYRIGHT
 
 Copyright 2000-2021 The OpenSSL Project Authors. All Rights Reserved.

doc/man1/openssl-s_server.pod.in

@@ -139,6 +139,7 @@ B<openssl> B<s_server>
 [B<-anti_replay>]
 [B<-no_anti_replay>]
 [B<-num_tickets>]
+[B<-tfo>]
 {- $OpenSSL::safe::opt_name_synopsis -}
 {- $OpenSSL::safe::opt_version_synopsis -}
 {- $OpenSSL::safe::opt_v_synopsis -}
@@ -816,6 +817,11 @@ has been negotiated, and early data is enabled on the server. A full handshake
 is forced if a session ticket is used a second or subsequent time. Any early
 data that was sent will be rejected.
 
+=item B<-tfo>
+
+Enable acceptance of TCP Fast Open (RFC7413) connections.
+
+
 {- $OpenSSL::safe::opt_name_item -}
 
 {- $OpenSSL::safe::opt_version_item -}
@@ -942,6 +948,8 @@ The
 The B<-srpvfile>, B<-srpuserseed>, and B<-engine>
 option were deprecated in OpenSSL 3.0.
 
+The -tfo option was added in OpenSSL 3.1.
+
 =head1 COPYRIGHT
 
 Copyright 2000-2021 The OpenSSL Project Authors. All Rights Reserved.

doc/man3/BIO_ADDR.pod

@@ -2,7 +2,8 @@
 
 =head1 NAME
 
-BIO_ADDR, BIO_ADDR_new, BIO_ADDR_clear, BIO_ADDR_free, BIO_ADDR_rawmake,
+BIO_ADDR, BIO_ADDR_new, BIO_ADDR_dup, BIO_ADDR_clear, BIO_ADDR_free,
+BIO_ADDR_rawmake,
 BIO_ADDR_family, BIO_ADDR_rawaddress, BIO_ADDR_rawport,
 BIO_ADDR_hostname_string, BIO_ADDR_service_string,
 BIO_ADDR_path_string - BIO_ADDR routines
@@ -15,6 +16,7 @@ BIO_ADDR_path_string - BIO_ADDR routines
  typedef union bio_addr_st BIO_ADDR;
 
  BIO_ADDR *BIO_ADDR_new(void);
+ BIO_ADDR *BIO_ADDR_dup(const BIO_ADDR *ap);
  void BIO_ADDR_free(BIO_ADDR *);
  void BIO_ADDR_clear(BIO_ADDR *ap);
  int BIO_ADDR_rawmake(BIO_ADDR *ap, int family,
@@ -37,7 +39,11 @@ BIO_ADDR_new() creates a new unfilled B<BIO_ADDR>, to be used
 with routines that will fill it with information, such as
 BIO_accept_ex().
 
-BIO_ADDR_free() frees a B<BIO_ADDR> created with BIO_ADDR_new().
+BIO_ADDR_dup() creates a new B<BIO_ADDR>, with a copy of the
+address data in B<ap>.
+
+BIO_ADDR_free() frees a B<BIO_ADDR> created with BIO_ADDR_new()
+or BIO_ADDR_dup();
 
 BIO_ADDR_clear() clears any data held within the provided B<BIO_ADDR> and sets
 it back to an uninitialised state.
@@ -113,6 +119,10 @@ information they should return isn't available.
 
 L<BIO_connect(3)>, L<BIO_s_connect(3)>
 
+=head1 HISTORY
+
+BIO_ADDR_dup() was added in OpenSSL 3.1.
+
 =head1 COPYRIGHT
 
 Copyright 2016-2020 The OpenSSL Project Authors. All Rights Reserved.

doc/man3/BIO_connect.pod

@@ -73,6 +73,15 @@ port.
 When creating an IPv6 socket, make it only listen for IPv6 addresses
 and not IPv4 addresses mapped to IPv6.
 
+=item BIO_SOCK_TFO
+
+Enables TCP Fast Open on the socket. Uses appropriate APIs on
+supported operating systems, including Linux, macOS and FreeBSD. Can
+be used with BIO_connect(), BIO_set_conn_mode(), BIO_set_bind_mode(),
+and BIO_listen().
+On Linux kernels before 4.14, use BIO_set_conn_address() to specify
+the peer address before starting the TLS handshake.
+
 =back
 
 These flags are bit flags, so they are to be combined with the

doc/man3/BIO_ctrl.pod

@@ -6,7 +6,7 @@ BIO_ctrl, BIO_callback_ctrl, BIO_ptr_ctrl, BIO_int_ctrl, BIO_reset,
 BIO_seek, BIO_tell, BIO_flush, BIO_eof, BIO_set_close, BIO_get_close,
 BIO_pending, BIO_wpending, BIO_ctrl_pending, BIO_ctrl_wpending,
 BIO_get_info_callback, BIO_set_info_callback, BIO_info_cb, BIO_get_ktls_send,
-BIO_get_ktls_recv
+BIO_get_ktls_recv, BIO_set_conn_mode, BIO_get_conn_mode, BIO_set_tfo
 - BIO control operations
 
 =head1 SYNOPSIS
@@ -38,6 +38,11 @@ BIO_get_ktls_recv
  int BIO_get_ktls_send(BIO *b);
  int BIO_get_ktls_recv(BIO *b);
 
+ int BIO_set_conn_mode(BIO *b, int mode);
+ int BIO_get_conn_mode(BIO *b);
+
+ int BIO_set_tfo(BIO *b, int onoff);
+
 =head1 DESCRIPTION
 
 BIO_ctrl(), BIO_callback_ctrl(), BIO_ptr_ctrl() and BIO_int_ctrl()
@@ -81,6 +86,13 @@ sending. Otherwise, it returns zero. It also returns negative values for failure
 BIO_get_ktls_recv() returns 1 if the BIO is using the Kernel TLS data-path for
 receiving. Otherwise, it returns zero. It also returns negative values for failure.
 
+BIO_get_conn_mode() returns the BIO connection mode. BIO_set_conn_mode() sets
+the BIO connection mode.
+
+BIO_set_tfo() disables TCP Fast Open when B<onoff> is 0, and enables TCP Fast
+Open when B<onoff> is nonzero. Setting the value to 1 is equivalent to setting
+B<BIO_SOCK_TFO> in BIO_set_conn_mode().
+
 =head1 RETURN VALUES
 
 BIO_reset() normally returns 1 for success and <=0 for failure. File
@@ -107,6 +119,19 @@ sending. Otherwise, it returns zero.
 BIO_get_ktls_recv() returns 1 if the BIO is using the Kernel TLS data-path for
 receiving. Otherwise, it returns zero.
 
+BIO_set_conn_mode() returns 1 for success and 0 for failure. BIO_get_conn_mode()
+returns the current connection mode. Which may contain the bitwise-or of the
+following flags:
+
+ BIO_SOCK_REUSEADDR
+ BIO_SOCK_V6_ONLY
+ BIO_SOCK_KEEPALIVE
+ BIO_SOCK_NONBLOCK
+ BIO_SOCK_NODELAY
+ BIO_SOCK_TFO
+
+BIO_set_tfo() returns 1 for success, and 0 for failure.
+
 =head1 NOTES
 
 BIO_flush(), because it can write data may return 0 or -1 indicating
@@ -144,6 +169,9 @@ the case of BIO_seek() on a file BIO for a successful operation.
 The BIO_get_ktls_send() and BIO_get_ktls_recv() functions were added in
 OpenSSL 3.0.
 
+The BIO_get_conn_mode(), BIO_set_conn_mode() and BIO_set_tfo() functions
+were added in OpenSSL 3.1.
+
 =head1 COPYRIGHT
 
 Copyright 2000-2021 The OpenSSL Project Authors. All Rights Reserved.

doc/man3/BIO_s_accept.pod

@@ -3,7 +3,7 @@
 =head1 NAME
 
 BIO_s_accept, BIO_set_accept_name, BIO_set_accept_port, BIO_get_accept_name,
-BIO_get_accept_port, BIO_new_accept, BIO_set_nbio_accept, BIO_set_accept_bios,
+BIO_get_accept_port, BIO_new_accept, BIO_set_nbio_accept, BIO_set_tfo_accept, BIO_set_accept_bios,
 BIO_get_peer_name, BIO_get_peer_port,
 BIO_get_accept_ip_family, BIO_set_accept_ip_family,
 BIO_set_bind_mode, BIO_get_bind_mode, BIO_do_accept - accept BIO
@@ -23,6 +23,7 @@ BIO_set_bind_mode, BIO_get_bind_mode, BIO_do_accept - accept BIO
  BIO *BIO_new_accept(char *host_port);
 
  long BIO_set_nbio_accept(BIO *b, int n);
+ long BIO_set_tfo_accept(BIO *b, int n);
  long BIO_set_accept_bios(BIO *b, char *bio);
 
  char *BIO_get_peer_name(BIO *b);
@@ -87,6 +88,11 @@ B<host_port>.
 BIO_set_nbio_accept() sets the accept socket to blocking mode
 (the default) if B<n> is 0 or non blocking mode if B<n> is 1.
 
+BIO_set_tfo_accept() enables TCP Fast Open on the accept socket
+if B<n> is 1 or disables TCP Fast Open if B<n> is 0 (the default).
+Setting the value to 1 is equivalent to setting B<BIO_SOCK_TFO>
+in BIO_set_bind_mode().
+
 BIO_set_accept_bios() can be used to set a chain of BIOs which
 will be duplicated and prepended to the chain when an incoming
 connection is received. This is useful if, for example, a
@@ -107,7 +113,9 @@ B<BIO_BIND_REUSEADDR> is set then other sockets can bind to the
 same port. If B<BIO_BIND_REUSEADDR_IF_UNUSED> is set then and
 attempt is first made to use BIO_BIN_NORMAL, if this fails
 and the port is not in use then a second attempt is made
-using B<BIO_BIND_REUSEADDR>.
+using B<BIO_BIND_REUSEADDR>. If B<BIO_SOCK_TFO> is set, then
+the socket will be configured to accept TCP Fast Open
+connections.
 
 BIO_do_accept() serves two functions. When it is first
 called, after the accept BIO has been setup, it will attempt
@@ -230,6 +238,10 @@ down each and finally closes both down.
  BIO_free(cbio);
  BIO_free(cbio2);
 
+=head1 HISTORY
+
+BIO_set_tfo_accept() was added in OpenSSL 3.1.
+
 =head1 COPYRIGHT
 
 Copyright 2000-2021 The OpenSSL Project Authors. All Rights Reserved.

doc/man7/bio.pod

@@ -36,7 +36,6 @@ BIO and one or more filter BIOs. Data read from or written to the
 first BIO then traverses the chain to the end (normally a source/sink
 BIO).
 
-
 Some BIOs (such as memory BIOs) can be used immediately after calling
 BIO_new(). Others (such as file BIOs) need some additional initialization,
 and frequently a utility function exists to create and initialize such BIOs.
@@ -52,6 +51,29 @@ pointer to a BIO_METHOD. There is a naming convention for such functions:
 a source/sink BIO typically starts with I<BIO_s_> and
 a filter BIO with I<BIO_f_>.
 
+=head2 TCP Fast Open
+
+TCP Fast Open (RFC7413), abbreviated "TFO", is supported by the BIO
+interface since OpenSSL 3.1. TFO is supported in the following operating systems:
+
+=over 4
+
+=item * Linux kernel 3.13 and later, where TFO is enabled by default.
+
+=item * Linux kernel 4.11 and later, using TCP_FASTOPEN_CONNECT.
+
+=item * FreeBSD 10.3 to 11.4, supports server TFO only.
+
+=item * FreeBSD 12.0 and later, supports both client and server TFO.
+
+=item * macOS 10.14 and later.
+
+=back
+
+Each operating system has a slightly different API for TFO. Please
+refer to the operating systems' API documentation when using
+sockets directly.
+
 =head1 EXAMPLES
 
 Create a memory BIO:
@@ -65,7 +87,9 @@ L<BIO_f_base64(3)>, L<BIO_f_buffer(3)>,
 L<BIO_f_cipher(3)>, L<BIO_f_md(3)>,
 L<BIO_f_null(3)>, L<BIO_f_ssl(3)>,
 L<BIO_f_readbuffer(3)>,
-L<BIO_find_type(3)>, L<BIO_new(3)>,
+L<BIO_find_type(3)>,
+L<BIO_get_conn_mode(3)>,
+L<BIO_new(3)>,
 L<BIO_new_bio_pair(3)>,
 L<BIO_push(3)>, L<BIO_read_ex(3)>,
 L<BIO_s_accept(3)>, L<BIO_s_bio(3)>,
@@ -73,6 +97,9 @@ L<BIO_s_connect(3)>, L<BIO_s_fd(3)>,
 L<BIO_s_file(3)>, L<BIO_s_mem(3)>,
 L<BIO_s_null(3)>, L<BIO_s_socket(3)>,
 L<BIO_set_callback(3)>,
+L<BIO_set_conn_mode(3)>,
+L<BIO_set_tfo(3)>,
+L<BIO_set_tfo_accept(3)>,
 L<BIO_should_retry(3)>
 
 =head1 COPYRIGHT
@@ -85,4 +112,3 @@ in the file LICENSE in the source distribution or at
 L<https://www.openssl.org/source/license.html>.
 
 =cut
-

include/internal/bio_tfo.h

@@ -0,0 +1,151 @@
+/*
+ * Copyright 2022 The OpenSSL Project Authors. All Rights Reserved.
+ *
+ * Licensed under the Apache License 2.0 (the "License").  You may not use
+ * this file except in compliance with the License.  You can obtain a copy
+ * in the file LICENSE in the source distribution or at
+ * https://www.openssl.org/source/license.html
+ */
+
+/*
+ * Contains definitions for simplifying the use of TCP Fast Open
+ * (RFC7413) in OpenSSL socket BIOs.
+ */
+
+/* If a supported OS is added here, update test/bio_tfo_test.c */
+#if defined(TCP_FASTOPEN) && !defined(OPENSSL_NO_TFO)
+
+# if defined(OPENSSL_SYS_MACOSX) || defined(__FreeBSD__)
+#  include <sys/sysctl.h>
+# endif
+
+/*
+ * OSSL_TFO_SYSCTL is used to determine if TFO is supported by
+ * this kernel, and if supported, if it is enabled. This is more of
+ * a problem on FreeBSD 10.3 ~ 11.4, where TCP_FASTOPEN was defined,
+ * but not enabled by default in the kernel, and only for the server.
+ * Linux does not have sysctlbyname(), and the closest equivalent
+ * is to go into the /proc filesystem, but I'm not sure it's
+ * worthwhile.
+ *
+ * On MacOS and Linux:
+ * These operating systems use a single parameter to control TFO.
+ * The OSSL_TFO_CLIENT_FLAG and OSSL_TFO_SERVER_FLAGS are used to
+ * determine if TFO is enabled for the client and server respectively.
+ *
+ * OSSL_TFO_CLIENT_FLAG = 1 = client TFO enabled
+ * OSSL_TFO_SERVER_FLAG = 2 = server TFO enabled
+ *
+ * Such that:
+ * 0 = TFO disabled
+ * 3 = server and client TFO enabled
+ *
+ * macOS 10.14 and later support TFO.
+ * Linux kernel 3.6 added support for client TFO.
+ * Linux kernel 3.7 added support for server TFO.
+ * Linux kernel 3.13 enabled TFO by default.
+ * Linux kernel 4.11 added the TCP_FASTOPEN_CONNECT option.
+ *
+ * On FreeBSD:
+ * FreeBSD 10.3 ~ 11.4 uses a single sysctl for server enable.
+ * FreeBSD 12.0 and later uses separate sysctls for server and
+ * client enable.
+ *
+ * Some options are purposely NOT defined per-platform
+ *
+ * OSSL_TFO_SYSCTL
+ *     Defined as a sysctlbyname() option to to determine if
+ *     TFO is enabled in the kernel (macOS, FreeBSD)
+ *
+ * OSSL_TFO_SERVER_SOCKOPT
+ *     Defined to indicate the socket option used to enable
+ *     TFO on a server socket (all)
+ *
+ * OSSL_TFO_SERVER_SOCKOPT_VALUE
+ *     Value to be used with OSSL_TFO_SERVER_SOCKOPT
+ *
+ * OSSL_TFO_CONNECTX
+ *     Use the connectx() function to make a client connection
+ *     (macOS)
+ *
+ * OSSL_TFO_CLIENT_SOCKOPT
+ *     Defined to indicate the socket option used to enable
+ *     TFO on a client socket (FreeBSD, Linux 4.14 and later)
+ *
+ * OSSL_TFO_SENDTO
+ *     Defined to indicate the sendto() message type to
+ *     be used to initiate a TFO connection (FreeBSD,
+ *     Linux pre-4.14)
+ *
+ * OSSL_TFO_DO_NOT_CONNECT
+ *     Defined to skip calling conect() when creating a
+ *     client socket (macOS, FreeBSD, Linux pre-4.14)
+ */
+
+# if defined(OPENSSL_SYS_WINDOWS)
+/*
+ * NO WINDOWS SUPPORT
+ *
+ * But this is is what would be used on the server:
+ *
+ * define OSSL_TFO_SERVER_SOCKOPT       TCP_FASTOPEN
+ * define OSSL_TFO_SERVER_SOCKOPT_VALUE 1
+ *
+ * Still have to figure out client support
+ */
+#  undef TCP_FASTOPEN
+# endif
+
+/* NO VMS SUPPORT */
+# if defined(OPENSSL_SYS_VMS)
+#  undef TCP_FASTOPEN
+# endif
+
+# if defined(OPENSSL_SYS_MACOSX)
+#  define OSSL_TFO_SYSCTL               "net.inet.tcp.fastopen"
+#  define OSSL_TFO_SERVER_SOCKOPT       TCP_FASTOPEN
+#  define OSSL_TFO_SERVER_SOCKOPT_VALUE 1
+#  define OSSL_TFO_CONNECTX             1
+#  define OSSL_TFO_DO_NOT_CONNECT       1
+#  define OSSL_TFO_CLIENT_FLAG          1
+#  define OSSL_TFO_SERVER_FLAG          2
+# endif
+
+# if defined(__FreeBSD__)
+#  if defined(TCP_FASTOPEN_PSK_LEN)
+/* As of 12.0 these are the SYSCTLs */
+#   define OSSL_TFO_SYSCTL_SERVER        "net.inet.tcp.fastopen.server_enable"
+#   define OSSL_TFO_SYSCTL_CLIENT        "net.inet.tcp.fastopen.client_enable"
+#   define OSSL_TFO_SERVER_SOCKOPT       TCP_FASTOPEN
+#   define OSSL_TFO_SERVER_SOCKOPT_VALUE MAX_LISTEN
+#   define OSSL_TFO_CLIENT_SOCKOPT       TCP_FASTOPEN
+#   define OSSL_TFO_DO_NOT_CONNECT       1
+#   define OSSL_TFO_SENDTO               0
+/* These are the same because the sysctl are client/server-specific */
+#   define OSSL_TFO_CLIENT_FLAG          1
+#   define OSSL_TFO_SERVER_FLAG          1
+#  else
+/* 10.3 through 11.4 SYSCTL - ONLY SERVER SUPPORT */
+#   define OSSL_TFO_SYSCTL               "net.inet.tcp.fastopen.enabled"
+#   define OSSL_TFO_SERVER_SOCKOPT       TCP_FASTOPEN
+#   define OSSL_TFO_SERVER_SOCKOPT_VALUE MAX_LISTEN
+#   define OSSL_TFO_SERVER_FLAG          1
+#  endif
+# endif
+
+# if defined(OPENSSL_SYS_LINUX)
+/* OSSL_TFO_PROC not used, but of interest */
+#  define OSSL_TFO_PROC                 "/proc/sys/net/ipv4/tcp_fastopen"
+#  define OSSL_TFO_SERVER_SOCKOPT       TCP_FASTOPEN
+#  define OSSL_TFO_SERVER_SOCKOPT_VALUE MAX_LISTEN
+#  if defined(TCP_FASTOPEN_CONNECT)
+#   define OSSL_TFO_CLIENT_SOCKOPT      TCP_FASTOPEN_CONNECT
+#  else
+#   define OSSL_TFO_SENDTO              MSG_FASTOPEN
+#   define OSSL_TFO_DO_NOT_CONNECT      1
+#  endif
+#  define OSSL_TFO_CLIENT_FLAG          1
+#  define OSSL_TFO_SERVER_FLAG          2
+# endif
+
+#endif

include/openssl/bio.h.in

@@ -405,10 +405,13 @@ struct bio_dgram_sctp_prinfo {
 
 # define BIO_C_SET_CONNECT_MODE                  155
 
+# define BIO_C_SET_TFO                           156 /* like BIO_C_SET_NBIO */
+
 # define BIO_set_app_data(s,arg)         BIO_set_ex_data(s,0,arg)
 # define BIO_get_app_data(s)             BIO_get_ex_data(s,0)
 
-# define BIO_set_nbio(b,n)             BIO_ctrl(b,BIO_C_SET_NBIO,(n),NULL)
+# define BIO_set_nbio(b,n)               BIO_ctrl(b,BIO_C_SET_NBIO,(n),NULL)
+# define BIO_set_tfo(b,n)                BIO_ctrl(b,BIO_C_SET_TFO,(n),NULL)
 
 # ifndef OPENSSL_NO_SOCK
 /* IP families we support, for BIO_s_connect() and BIO_s_accept() */
@@ -429,6 +432,7 @@ struct bio_dgram_sctp_prinfo {
 #  define BIO_get_conn_port(b)          ((const char *)BIO_ptr_ctrl(b,BIO_C_GET_CONNECT,1))
 #  define BIO_get_conn_address(b)       ((const BIO_ADDR *)BIO_ptr_ctrl(b,BIO_C_GET_CONNECT,2))
 #  define BIO_get_conn_ip_family(b)     BIO_ctrl(b,BIO_C_GET_CONNECT,3,NULL)
+#  define BIO_get_conn_mode(b)          BIO_ctrl(b,BIO_C_GET_CONNECT,4,NULL)
 #  define BIO_set_conn_mode(b,n)        BIO_ctrl(b,BIO_C_SET_CONNECT_MODE,(n),NULL)
 
 /* BIO_s_accept() */
@@ -446,6 +450,7 @@ struct bio_dgram_sctp_prinfo {
                                                  (char *)(bio))
 #  define BIO_set_accept_ip_family(b,f) BIO_int_ctrl(b,BIO_C_SET_ACCEPT,4,f)
 #  define BIO_get_accept_ip_family(b)   BIO_ctrl(b,BIO_C_GET_ACCEPT,4,NULL)
+#  define BIO_set_tfo_accept(b,n)       BIO_ctrl(b,BIO_C_SET_ACCEPT,5,(n)?(void *)"a":NULL)
 
 /* Aliases kept for backward compatibility */
 #  define BIO_BIND_NORMAL                 0
@@ -703,6 +708,7 @@ int BIO_hex_string(BIO *out, int indent, int width, const void *data,
 
 # ifndef OPENSSL_NO_SOCK
 BIO_ADDR *BIO_ADDR_new(void);
+BIO_ADDR *BIO_ADDR_dup(const BIO_ADDR *ap);
 int BIO_ADDR_rawmake(BIO_ADDR *ap, int family,
                      const void *where, size_t wherelen, unsigned short port);
 void BIO_ADDR_free(BIO_ADDR *);
@@ -765,6 +771,7 @@ int BIO_sock_info(int sock,
 #  define BIO_SOCK_KEEPALIVE    0x04
 #  define BIO_SOCK_NONBLOCK     0x08
 #  define BIO_SOCK_NODELAY      0x10
+#  define BIO_SOCK_TFO          0x20
 
 int BIO_socket(int domain, int socktype, int protocol, int options);
 int BIO_connect(int sock, const BIO_ADDR *addr, int options);

include/openssl/bioerr.h

@@ -45,6 +45,8 @@
 # define BIO_R_NO_PORT_DEFINED                            113
 # define BIO_R_NO_SUCH_FILE                               128
 # define BIO_R_NULL_PARAMETER                             115 /* unused */
+# define BIO_R_TFO_DISABLED                               106
+# define BIO_R_TFO_NO_KERNEL_SUPPORT                      108
 # define BIO_R_TRANSFER_ERROR                             104
 # define BIO_R_TRANSFER_TIMEOUT                           105
 # define BIO_R_UNABLE_TO_BIND_SOCKET                      117
@@ -53,6 +55,7 @@
 # define BIO_R_UNABLE_TO_LISTEN_SOCKET                    119
 # define BIO_R_UNABLE_TO_NODELAY                          138
 # define BIO_R_UNABLE_TO_REUSEADDR                        139
+# define BIO_R_UNABLE_TO_TFO                              109
 # define BIO_R_UNAVAILABLE_IP_FAMILY                      145
 # define BIO_R_UNINITIALIZED                              120
 # define BIO_R_UNKNOWN_INFO_TYPE                          140

test/bio_tfo_test.c

@@ -0,0 +1,418 @@
+/*
+ * Copyright 2022 The OpenSSL Project Authors. All Rights Reserved.
+ *
+ * Licensed under the Apache License 2.0 (the "License").  You may not use
+ * this file except in compliance with the License.  You can obtain a copy
+ * in the file LICENSE in the source distribution or at
+ * https://www.openssl.org/source/license.html
+ */
+
+#include <openssl/bio.h>
+#include "internal/e_os.h"
+#include "internal/sockets.h"
+#include "internal/bio_tfo.h"
+#include "testutil.h"
+
+/* If OS support is added in crypto/bio/bio_tfo.h, add it here */
+#if defined(OPENSSL_SYS_LINUX)
+# define GOOD_OS 1
+#elif defined(__FreeBSD__)
+# define GOOD_OS 1
+#elif defined(OPENSSL_SYS_MACOSX)
+# define GOOD_OS 1
+#else
+# ifdef GOOD_OS
+#  undef GOOD_OS
+# endif
+#endif
+
+#if !defined(OPENSSL_NO_TFO) && defined(GOOD_OS)
+
+/*
+ * This test is to ensure that if TCP Fast Open is configured, that socket
+ * connections will still work. These tests are able to detect if TCP Fast
+ * Open works, but the tests will pass as long as the socket connects.
+ *
+ * The first test function tests the socket interface as implemented as BIOs.
+ *
+ * The second test functions tests the socket interface as implemented as fds.
+ *
+ * The tests are run 5 times. The first time is without TFO.
+ * The second test will create the TCP fast open cookie,
+ * this can be seen in `ip tcp_metrics` and in /proc/net/netstat/ on Linux.
+ * e.g. on Linux 4.15.0-135-generic:
+ * $ grep '^TcpExt:' /proc/net/netstat | cut -d ' ' -f 84-90 | column -t
+ * The third attempt will use the cookie and actually do TCP fast open.
+ * The 4th time is client-TFO only, the 5th time is server-TFO only.
+ */
+
+#  define SOCKET_DATA "FooBar"
+#  define SOCKET_DATA_LEN sizeof(SOCKET_DATA)
+
+static int test_bio_tfo(int idx)
+{
+    BIO *cbio = NULL;
+    BIO *abio = NULL;
+    BIO *sbio = NULL;
+    int ret = 0;
+    int sockerr = 0;
+    const char *port;
+    int server_tfo = 0;
+    int client_tfo = 0;
+    size_t bytes;
+    char read_buffer[20];
+
+    switch (idx) {
+    default:
+    case 0:
+        break;
+    case 1:
+    case 2:
+        server_tfo = 1;
+        client_tfo = 1;
+        break;
+    case 3:
+        client_tfo = 1;
+        break;
+    case 4:
+        server_tfo = 1;
+        break;
+    }
+
+    /* ACCEPT SOCKET */
+    if (!TEST_ptr(abio = BIO_new_accept("localhost:0"))
+            || !TEST_true(BIO_set_nbio_accept(abio, 1))
+            || !TEST_true(BIO_set_tfo_accept(abio, server_tfo))
+            || !TEST_int_gt(BIO_do_accept(abio), 0)
+            || !TEST_ptr(port = BIO_get_accept_port(abio))) {
+        sockerr = get_last_socket_error();
+        goto err;
+    }
+
+    /* Note: first BIO_do_accept will basically do the bind/listen */
+
+    /* CLIENT SOCKET */
+    if (!TEST_ptr(cbio = BIO_new_connect("localhost"))
+            || !TEST_long_gt(BIO_set_conn_port(cbio, port), 0)
+            || !TEST_long_gt(BIO_set_nbio(cbio, 1), 0)
+            || !TEST_long_gt(BIO_set_tfo(cbio, client_tfo), 0)) {
+        sockerr = get_last_socket_error();
+        goto err;
+    }
+
+    /* FIRST ACCEPT: no connection should be established */
+    if (BIO_do_accept(abio) <= 0) {
+        if (!BIO_should_retry(abio)) {
+            sockerr = get_last_socket_error();
+            BIO_printf(bio_err, "Error: failed without EAGAIN\n");
+            goto err;
+        }
+    } else {
+        sbio = BIO_pop(abio);
+        BIO_printf(bio_err, "Error: accepted unknown connection\n");
+        goto err;
+    }
+
+    /* CONNECT ATTEMPT: different behavior based on TFO support */
+    if (BIO_do_connect(cbio) <= 0) {
+        sockerr = get_last_socket_error();
+        if (sockerr == EOPNOTSUPP) {
+            BIO_printf(bio_err, "Skip: TFO not enabled/supported for client\n");
+            goto success;
+        } else if (sockerr != EINPROGRESS) {
+            BIO_printf(bio_err, "Error: failed without EINPROGRESSn");
+            goto err;
+        }
+    }
+
+    /* macOS needs some time for this to happen, so put in a select */
+    if (!TEST_int_ge(BIO_wait(abio, time(NULL) + 2, 0), 0)) {
+        sockerr = get_last_socket_error();
+        BIO_printf(bio_err, "Error: socket wait failed\n");
+        goto err;
+    }
+
+    /* SECOND ACCEPT: if TFO is supported, this will still fail until data is sent */
+    if (BIO_do_accept(abio) <= 0) {
+        if (!BIO_should_retry(abio)) {
+            sockerr = get_last_socket_error();
+            BIO_printf(bio_err, "Error: failed without EAGAIN\n");
+            goto err;
+        }
+    } else {
+        if (idx == 0)
+            BIO_printf(bio_err, "Success: non-TFO connection accepted without data\n");
+        else if (idx == 1)
+            BIO_printf(bio_err, "Ignore: connection accepted before data, possibly no TFO cookie, or TFO may not be enabled\n");
+        else if (idx == 4)
+            BIO_printf(bio_err, "Success: connection accepted before data, client TFO is disabled\n");
+        else
+            BIO_printf(bio_err, "Warning: connection accepted before data, TFO may not be enabled\n");
+        sbio = BIO_pop(abio);
+        goto success;
+    }
+
+    /* SEND DATA: this should establish the actual TFO connection */
+    if (!TEST_true(BIO_write_ex(cbio, SOCKET_DATA, SOCKET_DATA_LEN, &bytes))) {
+        sockerr = get_last_socket_error();
+        goto err;
+    }
+
+    /* macOS needs some time for this to happen, so put in a select */
+    if (!TEST_int_ge(BIO_wait(abio, time(NULL) + 2, 0), 0)) {
+        sockerr = get_last_socket_error();
+        BIO_printf(bio_err, "Error: socket wait failed\n");
+        goto err;
+    }
+
+    /* FINAL ACCEPT: if TFO is enabled, socket should be accepted at *this* point */
+    if (BIO_do_accept(abio) <= 0) {
+        sockerr = get_last_socket_error();
+        BIO_printf(bio_err, "Error: socket not accepted\n");
+        goto err;
+    }
+    BIO_printf(bio_err, "Success: Server accepted socket after write\n");
+    if (!TEST_ptr(sbio = BIO_pop(abio))
+            || !TEST_true(BIO_read_ex(sbio, read_buffer, sizeof(read_buffer), &bytes))
+            || !TEST_size_t_eq(bytes, SOCKET_DATA_LEN)
+            || !TEST_strn_eq(read_buffer, SOCKET_DATA, SOCKET_DATA_LEN)) {
+        sockerr = get_last_socket_error();
+        goto err;
+    }
+
+success:
+    sockerr = 0;
+    ret = 1;
+
+err:
+    if (sockerr != 0) {
+        const char *errstr = strerror(sockerr);
+
+        if (errstr != NULL)
+            BIO_printf(bio_err, "last errno: %d=%s\n", sockerr, errstr);
+    }
+    BIO_free(cbio);
+    BIO_free(abio);
+    BIO_free(sbio);
+    return ret;
+}
+
+static int test_fd_tfo(int idx)
+{
+    struct sockaddr_storage sstorage;
+    socklen_t slen;
+    struct addrinfo *ai = NULL;
+    struct addrinfo hints;
+    int ret = 0;
+    int cfd = -1; /* client socket */
+    int afd = -1; /* accept socket */
+    int sfd = -1; /* server accepted socket */
+    BIO_ADDR *baddr = NULL;
+    char read_buffer[20];
+    int bytes_read;
+    int server_flags = BIO_SOCK_NONBLOCK;
+    int client_flags = BIO_SOCK_NONBLOCK;
+    int sockerr = 0;
+    unsigned short port;
+    void *addr;
+    size_t addrlen;
+
+    switch (idx) {
+    default:
+    case 0:
+        break;
+    case 1:
+    case 2:
+        server_flags |= BIO_SOCK_TFO;
+        client_flags |= BIO_SOCK_TFO;
+        break;
+    case 3:
+        client_flags |= BIO_SOCK_TFO;
+        break;
+    case 4:
+        server_flags |= BIO_SOCK_TFO;
+        break;
+    }
+
+    /* ADDRESS SETUP */
+    memset(&hints, 0, sizeof(hints));
+    hints.ai_family = AF_UNSPEC;
+    hints.ai_socktype = SOCK_STREAM;
+    if (!TEST_int_eq(getaddrinfo(NULL, "0", &hints, &ai), 0))
+        goto err;
+
+    switch (ai->ai_family) {
+        case AF_INET:
+            port = ((struct sockaddr_in *)ai->ai_addr)->sin_port;
+            addr = &((struct sockaddr_in *)ai->ai_addr)->sin_addr;
+            addrlen = sizeof(((struct sockaddr_in *)ai->ai_addr)->sin_addr);
+            BIO_printf(bio_err, "Using IPv4\n");
+            break;
+        case AF_INET6:
+            port = ((struct sockaddr_in6 *)ai->ai_addr)->sin6_port;
+            addr = &((struct sockaddr_in6 *)ai->ai_addr)->sin6_addr;
+            addrlen = sizeof(((struct sockaddr_in6 *)ai->ai_addr)->sin6_addr);
+            BIO_printf(bio_err, "Using IPv6\n");
+            break;
+        default:
+            BIO_printf(bio_err, "Unknown address family %d\n", ai->ai_family);
+            goto err;
+    }
+
+    if (!TEST_ptr(baddr = BIO_ADDR_new())
+            || !TEST_true(BIO_ADDR_rawmake(baddr, ai->ai_family, addr, addrlen, port)))
+        goto err;
+
+    /* ACCEPT SOCKET */
+
+    if (!TEST_int_ge(afd = BIO_socket(ai->ai_family, ai->ai_socktype, ai->ai_protocol, 0), 0)
+            || !TEST_true(BIO_listen(afd, baddr, server_flags)))
+        goto err;
+
+    /* UPDATE ADDRESS WITH PORT */
+    slen = sizeof(sstorage);
+    if (!TEST_int_ge(getsockname(afd, (struct sockaddr *)&sstorage, &slen), 0))
+        goto err;
+
+    switch (sstorage.ss_family) {
+        case AF_INET:
+            port = ((struct sockaddr_in *)&sstorage)->sin_port;
+            addr = &((struct sockaddr_in *)&sstorage)->sin_addr;
+            addrlen = sizeof(((struct sockaddr_in *)&sstorage)->sin_addr);
+            break;
+        case AF_INET6:
+            port = ((struct sockaddr_in6 *)&sstorage)->sin6_port;
+            addr = &((struct sockaddr_in6 *)&sstorage)->sin6_addr;
+            addrlen = sizeof(((struct sockaddr_in6 *)&sstorage)->sin6_addr);
+            break;
+        default:
+            goto err;
+    }
+
+    if(!TEST_true(BIO_ADDR_rawmake(baddr, sstorage.ss_family, addr, addrlen, port)))
+        goto err;
+
+    /* CLIENT SOCKET */
+    if (!TEST_int_ge(cfd = BIO_socket(ai->ai_family, ai->ai_socktype, ai->ai_protocol, 0), 0))
+        goto err;
+
+    /* FIRST ACCEPT: no connection should be established */
+    sfd = BIO_accept_ex(afd, NULL, 0);
+    if (sfd == -1) {
+        sockerr = get_last_socket_error();
+        /* Note: Windows would hit WSAEWOULDBLOCK */
+        if (sockerr != EAGAIN) {
+            BIO_printf(bio_err, "Error: failed without EAGAIN\n");
+            goto err;
+        }
+    } else {
+        BIO_printf(bio_err, "Error: accepted unknown connection\n");
+        goto err;
+    }
+
+    /* CONNECT ATTEMPT: different behavior based on TFO support */
+    if (!BIO_connect(cfd, baddr, client_flags)) {
+        sockerr = get_last_socket_error();
+        if (sockerr == EOPNOTSUPP) {
+            BIO_printf(bio_err, "Skip: TFO not enabled/supported for client\n");
+            goto success;
+        } else {
+            /* Note: Windows would hit WSAEWOULDBLOCK */
+            if (sockerr != EINPROGRESS) {
+                BIO_printf(bio_err, "Error: failed without EINPROGRESS\n");
+                goto err;
+            }
+        }
+    }
+
+    /* macOS needs some time for this to happen, so put in a select */
+    if (!TEST_int_ge(BIO_socket_wait(afd, 1, time(NULL) + 2), 0)) {
+        sockerr = get_last_socket_error();
+        BIO_printf(bio_err, "Error: socket wait failed\n");
+        goto err;
+    }
+
+    /* SECOND ACCEPT: if TFO is supported, this will still fail until data is sent */
+    sfd = BIO_accept_ex(afd, NULL, 0);
+    if (sfd == -1) {
+        sockerr = get_last_socket_error();
+        /* Note: Windows would hit WSAEWOULDBLOCK */
+        if (sockerr != EAGAIN) {
+            BIO_printf(bio_err, "Error: failed without EAGAIN\n");
+            goto err;
+        }
+    } else {
+        if (idx == 0)
+            BIO_printf(bio_err, "Success: non-TFO connection accepted without data\n");
+        else if (idx == 1)
+            BIO_printf(bio_err, "Ignore: connection accepted before data, possibly no TFO cookie, or TFO may not be enabled\n");
+        else if (idx == 4)
+            BIO_printf(bio_err, "Success: connection accepted before data, client TFO is disabled\n");
+        else
+            BIO_printf(bio_err, "Warning: connection accepted before data, TFO may not be enabled\n");
+        goto success;
+    }
+
+    /* SEND DATA: this should establish the actual TFO connection */
+#ifdef OSSL_TFO_SENDTO
+    if (!TEST_int_ge(sendto(cfd, SOCKET_DATA, SOCKET_DATA_LEN, OSSL_TFO_SENDTO,
+                            (struct sockaddr *)&sstorage, slen), 0)) {
+        sockerr = get_last_socket_error();
+        goto err;
+    }
+#else
+    if (!TEST_int_ge(writesocket(cfd, SOCKET_DATA, SOCKET_DATA_LEN), 0)) {
+        sockerr = get_last_socket_error();
+        goto err;
+    }
+#endif
+
+    /* macOS needs some time for this to happen, so put in a select */
+    if (!TEST_int_ge(BIO_socket_wait(afd, 1, time(NULL) + 2), 0)) {
+        sockerr = get_last_socket_error();
+        BIO_printf(bio_err, "Error: socket wait failed\n");
+        goto err;
+    }
+
+    /* FINAL ACCEPT: if TFO is enabled, socket should be accepted at *this* point */
+    sfd = BIO_accept_ex(afd, NULL, 0);
+    if (sfd == -1) {
+        sockerr = get_last_socket_error();
+        BIO_printf(bio_err, "Error: socket not accepted\n");
+        goto err;
+    }
+    BIO_printf(bio_err, "Success: Server accepted socket after write\n");
+    bytes_read = readsocket(sfd, read_buffer, sizeof(read_buffer));
+    if (!TEST_int_eq(bytes_read, SOCKET_DATA_LEN)
+            || !TEST_strn_eq(read_buffer, SOCKET_DATA, SOCKET_DATA_LEN)) {
+        sockerr = get_last_socket_error();
+        goto err;
+    }
+
+success:
+    sockerr = 0;
+    ret = 1;
+
+err:
+    if (sockerr != 0) {
+        const char *errstr = strerror(sockerr);
+
+        if (errstr != NULL)
+            BIO_printf(bio_err, "last errno: %d=%s\n", sockerr, errstr);
+    }
+    BIO_ADDR_free(baddr);
+    BIO_closesocket(cfd);
+    BIO_closesocket(sfd);
+    BIO_closesocket(afd);
+    return ret;
+}
+#endif
+
+int setup_tests(void)
+{
+#if !defined(OPENSSL_NO_TFO) && defined(GOOD_OS)
+    ADD_ALL_TESTS(test_bio_tfo, 5);
+    ADD_ALL_TESTS(test_fd_tfo, 5);
+#endif
+    return 1;
+}

test/build.info

@@ -62,7 +62,8 @@ IF[{- !$disabled{tests} -}]
           context_internal_test aesgcmtest params_test evp_pkey_dparams_test \
           keymgmt_internal_test hexstr_test provider_status_test defltfips_test \
           bio_readbuffer_test user_property_test pkcs7_test upcallstest \
-          provfetchtest prov_config_test rand_test ca_internals_test
+          provfetchtest prov_config_test rand_test ca_internals_test \
+          bio_tfo_test
 
   IF[{- !$disabled{'deprecated-3.0'} -}]
     PROGRAMS{noinst}=enginetest
@@ -370,6 +371,10 @@ IF[{- !$disabled{tests} -}]
   INCLUDE[bio_core_test]=../include ../apps/include
   DEPEND[bio_core_test]=../libcrypto libtestutil.a
 
+  SOURCE[bio_tfo_test]=bio_tfo_test.c
+  INCLUDE[bio_tfo_test]=../include ../apps/include ..
+  DEPEND[bio_tfo_test]=../libcrypto libtestutil.a
+
   SOURCE[params_api_test]=params_api_test.c
   INCLUDE[params_api_test]=../include ../apps/include
   DEPEND[params_api_test]=../libcrypto libtestutil.a

test/recipes/04-test_bio_tfo.t

@@ -0,0 +1,18 @@
+#! /usr/bin/env perl
+# Copyright 2016-2021 The OpenSSL Project Authors. All Rights Reserved.
+#
+# Licensed under the Apache License 2.0 (the "License").  You may not use
+# this file except in compliance with the License.  You can obtain a copy
+# in the file LICENSE in the source distribution or at
+# https://www.openssl.org/source/license.html
+
+use strict;
+use OpenSSL::Test;
+use OpenSSL::Test::Simple;
+use OpenSSL::Test::Utils;
+
+setup("test_bio_tfo");
+
+plan skip_all => "This test requires enable-tfo" if disabled("tfo");
+
+simple_test("test_bio_tfo", "bio_tfo_test");

test/recipes/82-test_tfo_cli.t

@@ -0,0 +1,87 @@
+#! /usr/bin/env perl
+# Copyright 2022 The OpenSSL Project Authors. All Rights Reserved.
+#
+# Licensed under the Apache License 2.0 (the "License").  You may not use
+# this file except in compliance with the License.  You can obtain a copy
+# in the file LICENSE in the source distribution or at
+# https://www.openssl.org/source/license.html
+
+use strict;
+use warnings;
+
+use IPC::Open2;
+use OpenSSL::Test qw/:DEFAULT srctop_file bldtop_file/;
+use OpenSSL::Test::Utils;
+
+setup("test_tfo");
+
+plan skip_all => "test_tfo_cli needs sock enabled" if disabled("sock");
+plan skip_all => "test_tfo_cli needs tls < 1.3 enabled"
+    if disabled("tls1") && disabled("tls1_1") && disabled("tls1_2");
+plan skip_all => "test_tfo_cli does not run on Windows nor VMS"
+    if $^O =~ /^(VMS|MSWin32|msys)$/;
+
+plan tests => 8;
+
+my $shlib_wrap = bldtop_file("util", "shlib_wrap.sh");
+my $apps_openssl = bldtop_file("apps", "openssl");
+my $cert = srctop_file("apps", "server.pem");
+
+sub run_test {
+    my $tfo = shift;
+
+    my $client_good = ! $tfo;
+    my $server_good = ! $tfo;
+    my $connect_good = 0;
+    my $port = "0";
+
+    # Not using TLSv1.3 allows the test to work with "no-ec"
+    my @s_cmd = ("s_server", "-accept", ":0", "-cert", $cert, "-www", "-no_tls1_3", "-naccept", "1");
+    push @s_cmd, "-tfo" if ($tfo);
+
+    my $spid = open2(my $sout, my $sin, $shlib_wrap, $apps_openssl, @s_cmd);
+
+    # Read until we get the port, TFO is output before the ACCEPT line
+    while (<$sout>) {
+        chomp;
+        $server_good = $tfo if /^Listening for TFO$/;
+        if (/^ACCEPT\s.*:(\d+)$/) {
+            $port = $1;
+            last;
+        }
+    }
+    print STDERR "Port: $port\n";
+    print STDERR "Invalid port\n" if ! ok($port);
+
+    # Start up the client
+    my @c_cmd = ("s_client", "-connect", ":$port", "-no_tls1_3");
+    push @c_cmd, "-tfo" if ($tfo);
+
+    my $cpid = open2(my $cout, my $cin, $shlib_wrap, $apps_openssl, @c_cmd);
+
+    # Do the "GET", which will cause the client to finish
+    print $cin "GET /\r\n";
+
+    waitpid($cpid, 0);
+    waitpid($spid, 0);
+
+    # Check the client output
+    while (<$cout>) {
+        chomp;
+        $client_good = $tfo if /^Connecting via TFO$/;
+        $connect_good = 1 if /^Content-type: text/;
+    }
+
+    print STDERR "Client TFO check failed\n" if ! ok($client_good);
+    print STDERR "Server TFO check failed\n" if ! ok($server_good);
+    print STDERR "Connection failed\n" if ! ok($connect_good);
+}
+
+for my $tfo (0..1) {
+    SKIP:
+    {
+        skip "TFO not enabled", 4 if disabled("tfo") && $tfo;
+
+        run_test($tfo);
+    }
+}

util/libcrypto.num

@@ -5436,3 +5436,4 @@ BN_signed_native2bn                     ?	3_1_0	EXIST::FUNCTION:
 BN_signed_bn2native                     ?	3_1_0	EXIST::FUNCTION:
 ASYNC_set_mem_functions                 ?	3_1_0	EXIST::FUNCTION:
 ASYNC_get_mem_functions                 ?	3_1_0	EXIST::FUNCTION:
+BIO_ADDR_dup                            ?	3_1_0	EXIST::FUNCTION:SOCK

util/missingmacro.txt

@@ -9,7 +9,6 @@ BIO_get_retry_flags(3)
 BIO_CB_return(3)
 BIO_cb_pre(3)
 BIO_cb_post(3)
-BIO_set_conn_mode(3)
 BIO_dup_state(3)
 BIO_buffer_get_num_lines(3)
 BIO_buffer_peek(3)

util/other.syms

@@ -166,6 +166,7 @@ BIO_get_conn_address                    define
 BIO_get_conn_hostname                   define
 BIO_get_conn_port                       define
 BIO_get_conn_ip_family                  define
+BIO_get_conn_mode                       define
 BIO_get_fd                              define
 BIO_get_fp                              define
 BIO_get_indent                          define
@@ -199,6 +200,7 @@ BIO_set_conn_address                    define
 BIO_set_conn_hostname                   define
 BIO_set_conn_port                       define
 BIO_set_conn_ip_family                  define
+BIO_set_conn_mode                       define
 BIO_set_fd                              define
 BIO_set_fp                              define
 BIO_set_indent                          define
@@ -214,6 +216,8 @@ BIO_set_ssl                             define
 BIO_set_ssl_mode                        define
 BIO_set_ssl_renegotiate_bytes           define
 BIO_set_ssl_renegotiate_timeout         define
+BIO_set_tfo                             define
+BIO_set_tfo_accept                      define
 BIO_set_write_buf_size                  define
 BIO_set_write_buffer_size               define
 BIO_should_io_special                   define