RT3955: Reduce some stack usage

Use malloc/free instead of big onstack buffers.

Reviewed-by: Tim Hudson <tjh@openssl.org>

crypto/bn/bn_prime.c

@@ -131,7 +131,7 @@
 static int witness(BIGNUM *w, const BIGNUM *a, const BIGNUM *a1,
                    const BIGNUM *a1_odd, int k, BN_CTX *ctx,
                    BN_MONT_CTX *mont);
-static int probable_prime(BIGNUM *rnd, int bits);
+static int probable_prime(BIGNUM *rnd, int bits, prime_t *mods);
 static int probable_prime_dh_safe(BIGNUM *rnd, int bits,
                                   const BIGNUM *add, const BIGNUM *rem,
                                   BN_CTX *ctx);
@@ -211,9 +211,13 @@ int BN_generate_prime_ex(BIGNUM *ret, int bits, int safe,
     BIGNUM *t;
     int found = 0;
     int i, j, c1 = 0;
-    BN_CTX *ctx;
+    BN_CTX *ctx = NULL;
+    prime_t *mods = NULL;
     int checks = BN_prime_checks_for_size(bits);
 
+    mods = OPENSSL_zalloc(sizeof(*mods) * NUMPRIMES);
+    if (mods == NULL)
+        goto err;
     if (bits < 2) {
         /* There are no prime numbers this small. */
         BNerr(BN_F_BN_GENERATE_PRIME_EX, BN_R_BITS_TOO_SMALL);
@@ -234,7 +238,7 @@ int BN_generate_prime_ex(BIGNUM *ret, int bits, int safe,
  loop:
     /* make a random number and set the top and bottom bits */
     if (add == NULL) {
-        if (!probable_prime(ret, bits))
+        if (!probable_prime(ret, bits, mods))
             goto err;
     } else {
         if (safe) {
@@ -285,6 +289,7 @@ int BN_generate_prime_ex(BIGNUM *ret, int bits, int safe,
     /* we have a prime :-) */
     found = 1;
  err:
+    OPENSSL_free(mods);
     if (ctx != NULL)
         BN_CTX_end(ctx);
     BN_CTX_free(ctx);
@@ -497,10 +502,9 @@ static int witness(BIGNUM *w, const BIGNUM *a, const BIGNUM *a1,
     return 1;
 }
 
-static int probable_prime(BIGNUM *rnd, int bits)
+static int probable_prime(BIGNUM *rnd, int bits, prime_t *mods)
 {
     int i;
-    prime_t mods[NUMPRIMES];
     BN_ULONG delta;
     BN_ULONG maxdelta = BN_MASK2 - primes[NUMPRIMES - 1];
     char is_single_word = bits <= BN_BITS2;

crypto/pkcs7/pk7_smime.c

@@ -64,6 +64,9 @@
 #include <openssl/x509.h>
 #include <openssl/x509v3.h>
 
+
+#define BUFFERSIZE 4096
+
 static int pkcs7_copy_existing_digest(PKCS7 *p7, PKCS7_SIGNER_INFO *si);
 
 PKCS7 *PKCS7_sign(X509 *signcert, EVP_PKEY *pkey, STACK_OF(X509) *certs,
@@ -113,6 +116,7 @@ int PKCS7_final(PKCS7 *p7, BIO *data, int flags)
 {
     BIO *p7bio;
     int ret = 0;
+
     if ((p7bio = PKCS7_dataInit(p7, NULL)) == NULL) {
         PKCS7err(PKCS7_F_PKCS7_FINAL, ERR_R_MALLOC_FAILURE);
         return 0;
@@ -253,7 +257,7 @@ int PKCS7_verify(PKCS7 *p7, STACK_OF(X509) *certs, X509_STORE *store,
     STACK_OF(PKCS7_SIGNER_INFO) *sinfos;
     PKCS7_SIGNER_INFO *si;
     X509_STORE_CTX cert_ctx;
-    char buf[4096];
+    char *buf = NULL;
     int i, j = 0, k, ret = 0;
     BIO *p7bio = NULL;
     BIO *tmpin = NULL, *tmpout = NULL;
@@ -355,8 +359,12 @@ int PKCS7_verify(PKCS7 *p7, STACK_OF(X509) *certs, X509_STORE *store,
         tmpout = out;
 
     /* We now have to 'read' from p7bio to calculate digests etc. */
+    if ((buf = OPENSSL_malloc(BUFFERSIZE)) == NULL) {
+        PKCS7err(PKCS7_F_PKCS7_VERIFY, ERR_R_MALLOC_FAILURE);
+        goto err;
+    }
     for (;;) {
-        i = BIO_read(p7bio, buf, sizeof(buf));
+        i = BIO_read(p7bio, buf, BUFFERSIZE);
         if (i <= 0)
             break;
         if (tmpout)
@@ -387,6 +395,7 @@ int PKCS7_verify(PKCS7 *p7, STACK_OF(X509) *certs, X509_STORE *store,
     ret = 1;
 
  err:
+    OPENSSL_free(buf);
     if (tmpin == indata) {
         if (indata)
             BIO_pop(p7bio);
@@ -505,7 +514,7 @@ int PKCS7_decrypt(PKCS7 *p7, EVP_PKEY *pkey, X509 *cert, BIO *data, int flags)
 {
     BIO *tmpmem;
     int ret, i;
-    char buf[4096];
+    char *buf = NULL;
 
     if (!p7) {
         PKCS7err(PKCS7_F_PKCS7_DECRYPT, PKCS7_R_INVALID_NULL_POINTER);
@@ -549,24 +558,29 @@ int PKCS7_decrypt(PKCS7 *p7, EVP_PKEY *pkey, X509 *cert, BIO *data, int flags)
         }
         BIO_free_all(bread);
         return ret;
-    } else {
-        for (;;) {
-            i = BIO_read(tmpmem, buf, sizeof(buf));
-            if (i <= 0) {
-                ret = 1;
-                if (BIO_method_type(tmpmem) == BIO_TYPE_CIPHER) {
-                    if (!BIO_get_cipher_status(tmpmem))
-                        ret = 0;
-                }
-
-                break;
-            }
-            if (BIO_write(data, buf, i) != i) {
-                ret = 0;
-                break;
-            }
-        }
-        BIO_free_all(tmpmem);
-        return ret;
     }
+    if ((buf = OPENSSL_malloc(BUFFERSIZE)) == NULL) {
+        PKCS7err(PKCS7_F_PKCS7_DECRYPT, ERR_R_MALLOC_FAILURE);
+        goto err;
+    }
+    for (;;) {
+        i = BIO_read(tmpmem, buf, BUFFERSIZE);
+        if (i <= 0) {
+            ret = 1;
+            if (BIO_method_type(tmpmem) == BIO_TYPE_CIPHER) {
+                if (!BIO_get_cipher_status(tmpmem))
+                    ret = 0;
+            }
+
+            break;
+        }
+        if (BIO_write(data, buf, i) != i) {
+            ret = 0;
+            break;
+        }
+    }
+err:
+    OPENSSL_free(buf);
+    BIO_free_all(tmpmem);
+    return ret;
 }