Make default_method mostly compile-time

Document thread-safety issues
Have RSA_null return NULL (always fails)

Reviewed-by: Matt Caswell <matt@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/2244)

CHANGES

@@ -57,6 +57,10 @@
   *) Support for SSL_OP_NO_ENCRYPT_THEN_MAC in SSL_CONF_cmd.
      [Emilia Käsper]
 
+  *) The RSA "null" method, which was partially supported to avoid patent
+     issues, has been replaced to always returns NULL.
+     [Rich Salz]
+
  Changes between 1.1.0d and 1.1.0e [16 Feb 2017]
 
   *) Encrypt-Then-Mac renegotiation crash

apps/speed.c

@@ -1444,12 +1444,8 @@ int speed_main(int argc, char **argv)
             continue;
         }
 #ifndef OPENSSL_NO_RSA
-# ifndef RSA_NULL
+        if (strcmp(*argv, "openssl") == 0)
-        if (strcmp(*argv, "openssl") == 0) {
-            RSA_set_default_method(RSA_PKCS1_OpenSSL());
             continue;
-        }
-# endif
         if (strcmp(*argv, "rsa") == 0) {
             rsa_doit[R_RSA_512] = rsa_doit[R_RSA_1024] =
                 rsa_doit[R_RSA_2048] = rsa_doit[R_RSA_3072] =

crypto/dh/dh_key.c

@@ -56,11 +56,23 @@ static DH_METHOD dh_ossl = {
     NULL
 };
 
+static const DH_METHOD *default_DH_method = &dh_ossl;
+
 const DH_METHOD *DH_OpenSSL(void)
 {
     return &dh_ossl;
 }
 
+void DH_set_default_method(const DH_METHOD *meth)
+{
+    default_DH_method = meth;
+}
+
+const DH_METHOD *DH_get_default_method(void)
+{
+    return default_DH_method;
+}
+
 static int generate_key(DH *dh)
 {
     int ok = 0;

crypto/dh/dh_lib.c

@@ -13,20 +13,6 @@
 #include "dh_locl.h"
 #include <openssl/engine.h>
 
-static const DH_METHOD *default_DH_method = NULL;
-
-void DH_set_default_method(const DH_METHOD *meth)
-{
-    default_DH_method = meth;
-}
-
-const DH_METHOD *DH_get_default_method(void)
-{
-    if (!default_DH_method)
-        default_DH_method = DH_OpenSSL();
-    return default_DH_method;
-}
-
 int DH_set_method(DH *dh, const DH_METHOD *meth)
 {
     /*

crypto/dsa/dsa_lib.c

@@ -17,20 +17,6 @@
 #include <openssl/engine.h>
 #include <openssl/dh.h>
 
-static const DSA_METHOD *default_DSA_method = NULL;
-
-void DSA_set_default_method(const DSA_METHOD *meth)
-{
-    default_DSA_method = meth;
-}
-
-const DSA_METHOD *DSA_get_default_method(void)
-{
-    if (!default_DSA_method)
-        default_DSA_method = DSA_OpenSSL();
-    return default_DSA_method;
-}
-
 DSA *DSA_new(void)
 {
     return DSA_new_method(NULL);

crypto/dsa/dsa_ossl.c

@@ -41,6 +41,18 @@ static DSA_METHOD openssl_dsa_meth = {
     NULL
 };
 
+static const DSA_METHOD *default_DSA_method = &openssl_dsa_meth;
+
+void DSA_set_default_method(const DSA_METHOD *meth)
+{
+    default_DSA_method = meth;
+}
+
+const DSA_METHOD *DSA_get_default_method(void)
+{
+    return default_DSA_method;
+}
+
 const DSA_METHOD *DSA_OpenSSL(void)
 {
     return &openssl_dsa_meth;

crypto/rsa/build.info

@@ -1,6 +1,6 @@
 LIBS=../../libcrypto
 SOURCE[../../libcrypto]=\
         rsa_ossl.c rsa_gen.c rsa_lib.c rsa_sign.c rsa_saos.c rsa_err.c \
-        rsa_pk1.c rsa_ssl.c rsa_none.c rsa_oaep.c rsa_chk.c rsa_null.c \
+        rsa_pk1.c rsa_ssl.c rsa_none.c rsa_oaep.c rsa_chk.c \
         rsa_pss.c rsa_x931.c rsa_asn1.c rsa_depr.c rsa_ameth.c rsa_prn.c \
         rsa_pmeth.c rsa_crpt.c rsa_x931g.c rsa_meth.c

crypto/rsa/rsa_err.c

@@ -41,11 +41,6 @@ static ERR_STRING_DATA RSA_str_functs[] = {
     {ERR_FUNC(RSA_F_RSA_METH_SET1_NAME), "RSA_meth_set1_name"},
     {ERR_FUNC(RSA_F_RSA_MGF1_TO_MD), "rsa_mgf1_to_md"},
     {ERR_FUNC(RSA_F_RSA_NEW_METHOD), "RSA_new_method"},
-    {ERR_FUNC(RSA_F_RSA_NULL), "RSA_NULL"},
-    {ERR_FUNC(RSA_F_RSA_NULL_PRIVATE_DECRYPT), "RSA_null_private_decrypt"},
-    {ERR_FUNC(RSA_F_RSA_NULL_PRIVATE_ENCRYPT), "RSA_null_private_encrypt"},
-    {ERR_FUNC(RSA_F_RSA_NULL_PUBLIC_DECRYPT), "RSA_null_public_decrypt"},
-    {ERR_FUNC(RSA_F_RSA_NULL_PUBLIC_ENCRYPT), "RSA_null_public_encrypt"},
     {ERR_FUNC(RSA_F_RSA_OSSL_PRIVATE_DECRYPT), "rsa_ossl_private_decrypt"},
     {ERR_FUNC(RSA_F_RSA_OSSL_PRIVATE_ENCRYPT), "rsa_ossl_private_encrypt"},
     {ERR_FUNC(RSA_F_RSA_OSSL_PUBLIC_DECRYPT), "rsa_ossl_public_decrypt"},

crypto/rsa/rsa_lib.c

@@ -17,31 +17,9 @@
 #include "internal/evp_int.h"
 #include "rsa_locl.h"
 
-static const RSA_METHOD *default_RSA_meth = NULL;
-
 RSA *RSA_new(void)
 {
-    RSA *r = RSA_new_method(NULL);
+    return RSA_new_method(NULL);
-
-    return r;
-}
-
-void RSA_set_default_method(const RSA_METHOD *meth)
-{
-    default_RSA_meth = meth;
-}
-
-const RSA_METHOD *RSA_get_default_method(void)
-{
-    if (default_RSA_meth == NULL) {
-#ifdef RSA_NULL
-        default_RSA_meth = RSA_null_method();
-#else
-        default_RSA_meth = RSA_PKCS1_OpenSSL();
-#endif
-    }
-
-    return default_RSA_meth;
 }
 
 const RSA_METHOD *RSA_get_method(const RSA *rsa)

crypto/rsa/rsa_null.c

@@ -1,93 +0,0 @@
-/*
- * Copyright 1999-2016 The OpenSSL Project Authors. All Rights Reserved.
- *
- * Licensed under the OpenSSL license (the "License").  You may not use
- * this file except in compliance with the License.  You can obtain a copy
- * in the file LICENSE in the source distribution or at
- * https://www.openssl.org/source/license.html
- */
-
-#include <stdio.h>
-#include "internal/cryptlib.h"
-#include <openssl/bn.h>
-#include "rsa_locl.h"
-
-/*
- * This is a dummy RSA implementation that just returns errors when called.
- * It is designed to allow some RSA functions to work while stopping those
- * covered by the RSA patent. That is RSA, encryption, decryption, signing
- * and verify is not allowed but RSA key generation, key checking and other
- * operations (like storing RSA keys) are permitted.
- */
-
-static int RSA_null_public_encrypt(int flen, const unsigned char *from,
-                                   unsigned char *to, RSA *rsa, int padding);
-static int RSA_null_private_encrypt(int flen, const unsigned char *from,
-                                    unsigned char *to, RSA *rsa, int padding);
-static int RSA_null_public_decrypt(int flen, const unsigned char *from,
-                                   unsigned char *to, RSA *rsa, int padding);
-static int RSA_null_private_decrypt(int flen, const unsigned char *from,
-                                    unsigned char *to, RSA *rsa, int padding);
-static int RSA_null_init(RSA *rsa);
-static int RSA_null_finish(RSA *rsa);
-static RSA_METHOD rsa_null_meth = {
-    "Null RSA",
-    RSA_null_public_encrypt,
-    RSA_null_public_decrypt,
-    RSA_null_private_encrypt,
-    RSA_null_private_decrypt,
-    NULL,
-    NULL,
-    RSA_null_init,
-    RSA_null_finish,
-    0,
-    NULL,
-    NULL,
-    NULL,
-    NULL
-};
-
-const RSA_METHOD *RSA_null_method(void)
-{
-    return (&rsa_null_meth);
-}
-
-static int RSA_null_public_encrypt(int flen, const unsigned char *from,
-                                   unsigned char *to, RSA *rsa, int padding)
-{
-    RSAerr(RSA_F_RSA_NULL_PUBLIC_ENCRYPT, RSA_R_RSA_OPERATIONS_NOT_SUPPORTED);
-    return -1;
-}
-
-static int RSA_null_private_encrypt(int flen, const unsigned char *from,
-                                    unsigned char *to, RSA *rsa, int padding)
-{
-    RSAerr(RSA_F_RSA_NULL_PRIVATE_ENCRYPT,
-           RSA_R_RSA_OPERATIONS_NOT_SUPPORTED);
-    return -1;
-}
-
-static int RSA_null_private_decrypt(int flen, const unsigned char *from,
-                                    unsigned char *to, RSA *rsa, int padding)
-{
-    RSAerr(RSA_F_RSA_NULL_PRIVATE_DECRYPT,
-           RSA_R_RSA_OPERATIONS_NOT_SUPPORTED);
-    return -1;
-}
-
-static int RSA_null_public_decrypt(int flen, const unsigned char *from,
-                                   unsigned char *to, RSA *rsa, int padding)
-{
-    RSAerr(RSA_F_RSA_NULL_PUBLIC_DECRYPT, RSA_R_RSA_OPERATIONS_NOT_SUPPORTED);
-    return -1;
-}
-
-static int RSA_null_init(RSA *rsa)
-{
-    return (1);
-}
-
-static int RSA_null_finish(RSA *rsa)
-{
-    return (1);
-}

crypto/rsa/rsa_ossl.c

@@ -11,8 +11,6 @@
 #include "internal/bn_int.h"
 #include "rsa_locl.h"
 
-#ifndef RSA_NULL
-
 static int rsa_ossl_public_encrypt(int flen, const unsigned char *from,
                                   unsigned char *to, RSA *rsa, int padding);
 static int rsa_ossl_private_encrypt(int flen, const unsigned char *from,
@@ -26,7 +24,7 @@ static int rsa_ossl_mod_exp(BIGNUM *r0, const BIGNUM *i, RSA *rsa,
 static int rsa_ossl_init(RSA *rsa);
 static int rsa_ossl_finish(RSA *rsa);
 static RSA_METHOD rsa_pkcs1_ossl_meth = {
-    "OpenSSL PKCS#1 RSA (from Eric Young)",
+    "OpenSSL PKCS#1 RSA",
     rsa_ossl_public_encrypt,
     rsa_ossl_public_decrypt,     /* signature verification */
     rsa_ossl_private_encrypt,    /* signing */
@@ -43,11 +41,28 @@ static RSA_METHOD rsa_pkcs1_ossl_meth = {
     NULL                        /* rsa_keygen */
 };
 
+static const RSA_METHOD *default_RSA_meth = &rsa_pkcs1_ossl_meth;
+
+void RSA_set_default_method(const RSA_METHOD *meth)
+{
+    default_RSA_meth = meth;
+}
+
+const RSA_METHOD *RSA_get_default_method(void)
+{
+    return default_RSA_meth;
+}
+
 const RSA_METHOD *RSA_PKCS1_OpenSSL(void)
 {
     return &rsa_pkcs1_ossl_meth;
 }
 
+const RSA_METHOD *RSA_null_method(void)
+{
+    return NULL;
+}
+
 static int rsa_ossl_public_encrypt(int flen, const unsigned char *from,
                                   unsigned char *to, RSA *rsa, int padding)
 {
@@ -786,5 +801,3 @@ static int rsa_ossl_finish(RSA *rsa)
     BN_MONT_CTX_free(rsa->_method_mod_q);
     return (1);
 }
-
-#endif

crypto/ui/ui_lib.c

@@ -15,8 +15,6 @@
 #include <openssl/err.h>
 #include "ui_locl.h"
 
-static const UI_METHOD *default_UI_meth = NULL;
-
 UI *UI_new(void)
 {
     return (UI_new_method(NULL));
@@ -535,19 +533,6 @@ void *UI_get_ex_data(UI *r, int idx)
     return (CRYPTO_get_ex_data(&r->ex_data, idx));
 }
 
-void UI_set_default_method(const UI_METHOD *meth)
-{
-    default_UI_meth = meth;
-}
-
-const UI_METHOD *UI_get_default_method(void)
-{
-    if (default_UI_meth == NULL) {
-        default_UI_meth = UI_OpenSSL();
-    }
-    return default_UI_meth;
-}
-
 const UI_METHOD *UI_get_method(UI *ui)
 {
     return ui->meth;

crypto/ui/ui_openssl.c

@@ -202,6 +202,18 @@ static UI_METHOD ui_openssl = {
     NULL
 };
 
+static const UI_METHOD *default_UI_meth = &ui_openssl;
+
+void UI_set_default_method(const UI_METHOD *meth)
+{
+    default_UI_meth = meth;
+}
+
+const UI_METHOD *UI_get_default_method(void)
+{
+    return default_UI_meth;
+}
+
 /* The method with all the built-in thingies */
 UI_METHOD *UI_OpenSSL(void)
 {

doc/man3/DH_set_method.pod

@@ -31,8 +31,11 @@ Initially, the default DH_METHOD is the OpenSSL internal implementation, as
 returned by DH_OpenSSL().
 
 DH_set_default_method() makes B<meth> the default method for all DH
-structures created later. B<NB>: This is true only whilst no ENGINE has been set
+structures created later.
+B<NB>: This is true only whilst no ENGINE has been set
 as a default for DH, so this function is no longer recommended.
+This function is not thread-safe and should not be called at the same time
+as other OpenSSL functions.
 
 DH_get_default_method() returns a pointer to the current default DH_METHOD.
 However, the meaningfulness of this result is dependent on whether the ENGINE

doc/man3/DSA_set_method.pod

@@ -31,8 +31,11 @@ Initially, the default DSA_METHOD is the OpenSSL internal implementation,
 as returned by DSA_OpenSSL().
 
 DSA_set_default_method() makes B<meth> the default method for all DSA
-structures created later. B<NB>: This is true only whilst no ENGINE has
+structures created later.
+B<NB>: This is true only whilst no ENGINE has
 been set as a default for DSA, so this function is no longer recommended.
+This function is not thread-safe and should not be called at the same time
+as other OpenSSL functions.
 
 DSA_get_default_method() returns a pointer to the current default
 DSA_METHOD. However, the meaningfulness of this result is dependent on

doc/man3/RSA_set_method.pod

@@ -3,7 +3,7 @@
 =head1 NAME
 
 RSA_set_default_method, RSA_get_default_method, RSA_set_method,
-RSA_get_method, RSA_PKCS1_OpenSSL, RSA_null_method, RSA_flags,
+RSA_get_method, RSA_PKCS1_OpenSSL, RSA_flags,
 RSA_new_method - select RSA method
 
 =head1 SYNOPSIS
@@ -20,8 +20,6 @@ RSA_new_method - select RSA method
 
  RSA_METHOD *RSA_PKCS1_OpenSSL(void);
 
- RSA_METHOD *RSA_null_method(void);
-
  int RSA_flags(const RSA *rsa);
 
  RSA *RSA_new_method(ENGINE *engine);
@@ -38,8 +36,11 @@ Initially, the default RSA_METHOD is the OpenSSL internal implementation,
 as returned by RSA_PKCS1_OpenSSL().
 
 RSA_set_default_method() makes B<meth> the default method for all RSA
-structures created later. B<NB>: This is true only whilst no ENGINE has
+structures created later.
+B<NB>: This is true only whilst no ENGINE has
 been set as a default for RSA, so this function is no longer recommended.
+This function is not thread-safe and should not be called at the same time
+as other OpenSSL functions.
 
 RSA_get_default_method() returns a pointer to the current default
 RSA_METHOD. However, the meaningfulness of this result is dependent on
@@ -168,6 +169,11 @@ not currently exist).
 
 L<RSA_new(3)>
 
+=head1 HISTORY
+
+The RSA_null_method(), which was a partial attempt to avoid patent issues,
+was replaced to always return NULL in OpenSSL 1.1.1.
+
 =head1 COPYRIGHT
 
 Copyright 2000-2016 The OpenSSL Project Authors. All Rights Reserved.

doc/man3/UI_new.pod

@@ -168,6 +168,8 @@ B<UI_CTRL_IS_REDOABLE>, which returns a flag saying if the used UI can
 be used again or not.
 
 UI_set_default_method() changes the default UI method to the one given.
+This function is not thread-safe and should not be called at the same time
+as other OpenSSL functions.
 
 UI_get_default_method() returns a pointer to the current default UI method.
 

fuzz/client.c

@@ -36,16 +36,6 @@ int FuzzerInitialize(int *argc, char ***argv)
     idx = SSL_get_ex_data_X509_STORE_CTX_idx();
     RAND_add("", 1, ENTROPY_NEEDED);
     RAND_status();
-    RSA_get_default_method();
-#ifndef OPENSSL_NO_DSA
-    DSA_get_default_method();
-#endif
-#ifndef OPENSSL_NO_EC
-    EC_KEY_get_default_method();
-#endif
-#ifndef OPENSSL_NO_DH
-    DH_get_default_method();
-#endif
     comp_methods = SSL_COMP_get_compression_methods();
     OPENSSL_sk_sort((OPENSSL_STACK *)comp_methods);
 

fuzz/server.c

@@ -484,16 +484,6 @@ int FuzzerInitialize(int *argc, char ***argv)
     idx = SSL_get_ex_data_X509_STORE_CTX_idx();
     RAND_add("", 1, ENTROPY_NEEDED);
     RAND_status();
-    RSA_get_default_method();
-#ifndef OPENSSL_NO_DSA
-    DSA_get_default_method();
-#endif
-#ifndef OPENSSL_NO_EC
-    EC_KEY_get_default_method();
-#endif
-#ifndef OPENSSL_NO_DH
-    DH_get_default_method();
-#endif
     comp_methods = SSL_COMP_get_compression_methods();
     OPENSSL_sk_sort((OPENSSL_STACK *)comp_methods);
 

include/openssl/rsa.h

@@ -236,14 +236,13 @@ int RSA_flags(const RSA *r);
 
 void RSA_set_default_method(const RSA_METHOD *meth);
 const RSA_METHOD *RSA_get_default_method(void);
+const RSA_METHOD *RSA_null_method(void);
 const RSA_METHOD *RSA_get_method(const RSA *rsa);
 int RSA_set_method(RSA *rsa, const RSA_METHOD *meth);
 
 /* these are the actual RSA functions */
 const RSA_METHOD *RSA_PKCS1_OpenSSL(void);
 
-const RSA_METHOD *RSA_null_method(void);
-
 int RSA_pkey_ctx_ctrl(EVP_PKEY_CTX *ctx, int optype, int cmd, int p1, void *p2);
 
 DECLARE_ASN1_ENCODE_FUNCTIONS_const(RSA, RSAPublicKey)
@@ -502,11 +501,11 @@ int ERR_load_RSA_strings(void);
 # define RSA_F_RSA_METH_SET1_NAME                         163
 # define RSA_F_RSA_MGF1_TO_MD                             157
 # define RSA_F_RSA_NEW_METHOD                             106
-# define RSA_F_RSA_NULL                                   124
+# define RSA_F_RSA_NULL                                   0
-# define RSA_F_RSA_NULL_PRIVATE_DECRYPT                   132
+# define RSA_F_RSA_NULL_PRIVATE_DECRYPT                   0
-# define RSA_F_RSA_NULL_PRIVATE_ENCRYPT                   133
+# define RSA_F_RSA_NULL_PRIVATE_ENCRYPT                   0
-# define RSA_F_RSA_NULL_PUBLIC_DECRYPT                    134
+# define RSA_F_RSA_NULL_PUBLIC_DECRYPT                    0
-# define RSA_F_RSA_NULL_PUBLIC_ENCRYPT                    135
+# define RSA_F_RSA_NULL_PUBLIC_ENCRYPT                    0
 # define RSA_F_RSA_OSSL_PRIVATE_DECRYPT                   101
 # define RSA_F_RSA_OSSL_PRIVATE_ENCRYPT                   102
 # define RSA_F_RSA_OSSL_PUBLIC_DECRYPT                    103