Move OS-specific fopen quirks to o_fopen.c.

Reviewed-by: Richard Levitte <levitte@openssl.org>

crypto/bio/bss_file.c

@@ -61,91 +61,10 @@ static const BIO_METHOD methods_filep = {
     NULL,
 };
 
-static FILE *file_fopen(const char *filename, const char *mode)
-{
-    FILE *file = NULL;
-
-#  if defined(_WIN32) && defined(CP_UTF8)
-    int sz, len_0 = (int)strlen(filename) + 1;
-    DWORD flags;
-
-    /*
-     * Basically there are three cases to cover: a) filename is
-     * pure ASCII string; b) actual UTF-8 encoded string and
-     * c) locale-ized string, i.e. one containing 8-bit
-     * characters that are meaningful in current system locale.
-     * If filename is pure ASCII or real UTF-8 encoded string,
-     * MultiByteToWideChar succeeds and _wfopen works. If
-     * filename is locale-ized string, chances are that
-     * MultiByteToWideChar fails reporting
-     * ERROR_NO_UNICODE_TRANSLATION, in which case we fall
-     * back to fopen...
-     */
-    if ((sz = MultiByteToWideChar(CP_UTF8, (flags = MB_ERR_INVALID_CHARS),
-                                  filename, len_0, NULL, 0)) > 0 ||
-        (GetLastError() == ERROR_INVALID_FLAGS &&
-         (sz = MultiByteToWideChar(CP_UTF8, (flags = 0),
-                                   filename, len_0, NULL, 0)) > 0)
-        ) {
-        WCHAR wmode[8];
-        WCHAR *wfilename = _alloca(sz * sizeof(WCHAR));
-
-        if (MultiByteToWideChar(CP_UTF8, flags,
-                                filename, len_0, wfilename, sz) &&
-            MultiByteToWideChar(CP_UTF8, 0, mode, strlen(mode) + 1,
-                                wmode, OSSL_NELEM(wmode)) &&
-            (file = _wfopen(wfilename, wmode)) == NULL &&
-            (errno == ENOENT || errno == EBADF)
-            ) {
-            /*
-             * UTF-8 decode succeeded, but no file, filename
-             * could still have been locale-ized...
-             */
-            file = fopen(filename, mode);
-        }
-    } else if (GetLastError() == ERROR_NO_UNICODE_TRANSLATION) {
-        file = fopen(filename, mode);
-    }
-#  elif defined(__DJGPP__)
-    {
-        char *newname = NULL;
-
-        if (!HAS_LFN_SUPPORT(filename)) {
-            char *iterator;
-            char lastchar;
-
-            newname = OPENSSL_malloc(strlen(filename) + 1);
-            if (newname == NULL)
-                return NULL;
-
-            for(iterator = newname, lastchar = '\0';
-                *filename; filename++, iterator++) {
-                if (lastchar == '/' && filename[0] == '.'
-                    && filename[1] != '.' && filename[1] != '/') {
-                    /* Leading dots are not permitted in plain DOS. */
-                    *iterator = '_';
-                } else {
-                    *iterator = *filename;
-                }
-                lastchar = *filename;
-            }
-            *iterator = '\0';
-            filename = newname;
-        }
-        file = fopen(filename, mode);
-
-        OPENSSL_free(newname);
-    }
-#  else
-    file = fopen(filename, mode);
-#  endif
-    return (file);
-}
-
 BIO *BIO_new_file(const char *filename, const char *mode)
 {
     BIO  *ret;
-    FILE *file = file_fopen(filename, mode);
+    FILE *file = openssl_fopen(filename, mode);
     int fp_flags = BIO_CLOSE;
 
     if (strchr(mode, 'b') == NULL)
@@ -363,7 +282,7 @@ static long file_ctrl(BIO *b, int cmd, long num, void *ptr)
         else
             strcat(p, "t");
 #  endif
-        fp = file_fopen(ptr, p);
+        fp = openssl_fopen(ptr, p);
         if (fp == NULL) {
             SYSerr(SYS_F_FOPEN, get_last_sys_error());
             ERR_add_error_data(5, "fopen('", ptr, "','", p, "')");

crypto/build.info

@@ -2,7 +2,7 @@
 LIBS=../libcrypto
 SOURCE[../libcrypto]=\
         cryptlib.c mem.c mem_dbg.c cversion.c ex_data.c cpt_err.c \
-        ebcdic.c uid.c o_time.c o_str.c o_dir.c \
+        ebcdic.c uid.c o_time.c o_str.c o_dir.c o_fopen.c \
         threads_pthread.c threads_win.c threads_none.c \
         o_init.c o_fips.c mem_sec.c init.c {- $target{cpuid_asm_src} -} \
         {- $target{uplink_aux_src} -}

crypto/include/internal/cryptlib.h

@@ -25,7 +25,6 @@
 # include <openssl/buffer.h>
 # include <openssl/bio.h>
 # include <openssl/err.h>
-# include <openssl/opensslconf.h>
 
 #ifdef  __cplusplus
 extern "C" {
@@ -69,6 +68,11 @@ extern int OPENSSL_NONPIC_relocated;
 void crypto_cleanup_all_ex_data_int(void);
 
 int openssl_strerror_r(int errnum, char *buf, size_t buflen);
+# if !defined(OPENSSL_NO_STDIO)
+FILE *openssl_fopen(const char *filename, const char *mode);
+# else
+void *openssl_fopen(const char *filename, const char *mode);
+# endif
 
 #ifdef  __cplusplus
 }

crypto/o_fopen.c

@@ -0,0 +1,103 @@
+/*
+ * Copyright 2016 The OpenSSL Project Authors. All Rights Reserved.
+ *
+ * Licensed under the OpenSSL license (the "License").  You may not use
+ * this file except in compliance with the License.  You can obtain a copy
+ * in the file LICENSE in the source distribution or at
+ * https://www.openssl.org/source/license.html
+ */
+
+#include "internal/cryptlib.h"
+
+#if !defined(OPENSSL_NO_STDIO)
+
+# include <stdio.h>
+
+FILE *openssl_fopen(const char *filename, const char *mode)
+{
+    FILE *file = NULL;
+# if defined(_WIN32) && defined(CP_UTF8)
+    int sz, len_0 = (int)strlen(filename) + 1;
+    DWORD flags;
+
+    /*
+     * Basically there are three cases to cover: a) filename is
+     * pure ASCII string; b) actual UTF-8 encoded string and
+     * c) locale-ized string, i.e. one containing 8-bit
+     * characters that are meaningful in current system locale.
+     * If filename is pure ASCII or real UTF-8 encoded string,
+     * MultiByteToWideChar succeeds and _wfopen works. If
+     * filename is locale-ized string, chances are that
+     * MultiByteToWideChar fails reporting
+     * ERROR_NO_UNICODE_TRANSLATION, in which case we fall
+     * back to fopen...
+     */
+    if ((sz = MultiByteToWideChar(CP_UTF8, (flags = MB_ERR_INVALID_CHARS),
+                                  filename, len_0, NULL, 0)) > 0 ||
+        (GetLastError() == ERROR_INVALID_FLAGS &&
+         (sz = MultiByteToWideChar(CP_UTF8, (flags = 0),
+                                   filename, len_0, NULL, 0)) > 0)
+        ) {
+        WCHAR wmode[8];
+        WCHAR *wfilename = _alloca(sz * sizeof(WCHAR));
+
+        if (MultiByteToWideChar(CP_UTF8, flags,
+                                filename, len_0, wfilename, sz) &&
+            MultiByteToWideChar(CP_UTF8, 0, mode, strlen(mode) + 1,
+                                wmode, OSSL_NELEM(wmode)) &&
+            (file = _wfopen(wfilename, wmode)) == NULL &&
+            (errno == ENOENT || errno == EBADF)
+            ) {
+            /*
+             * UTF-8 decode succeeded, but no file, filename
+             * could still have been locale-ized...
+             */
+            file = fopen(filename, mode);
+        }
+    } else if (GetLastError() == ERROR_NO_UNICODE_TRANSLATION) {
+        file = fopen(filename, mode);
+    }
+# elif defined(__DJGPP__)
+    {
+        char *newname = NULL;
+
+        if (!HAS_LFN_SUPPORT(filename)) {
+            char *iterator;
+            char lastchar;
+
+            newname = OPENSSL_malloc(strlen(filename) + 1);
+            if (newname == NULL)
+                return NULL;
+
+            for(iterator = newname, lastchar = '\0';
+                *filename; filename++, iterator++) {
+                if (lastchar == '/' && filename[0] == '.'
+                    && filename[1] != '.' && filename[1] != '/') {
+                    /* Leading dots are not permitted in plain DOS. */
+                    *iterator = '_';
+                } else {
+                    *iterator = *filename;
+                }
+                lastchar = *filename;
+            }
+            *iterator = '\0';
+            filename = newname;
+        }
+        file = fopen(filename, mode);
+
+        OPENSSL_free(newname);
+    }
+# else
+    file = fopen(filename, mode);
+# endif
+    return file;
+}
+
+#else
+
+void *openssl_fopen(const char *filename, const char *mode)
+{
+    return NULL;
+}
+
+#endif