bug 1337950 - work around failing to load a FIPS PKCS#11 module DB in NSS initialization r=Cykesiopka,jcj

Firefox essentially does not support running NSS in FIPS mode any longer. This
has always been the case on Android from what I can tell and it has been the
case on OS X since at least version 34 (see bug 1047584). It became the case on
Windows as of version 53 (see bug 1295937). Unfortunately, before this patch,
if a user attempted to run an affected version of Firefox using a profile
directory containing an NSS database collection that had FIPS enabled, NSS
initialization would fail and fall back to running in no DB mode, which had the
side-effect of making any saved passwords and certificates unavailable. This
patch attempts to detect and work around this failure mode by moving the
PKCS#11 module DB (which is where the FIPS bit is set) to a backup location and
basically running with a fresh, non-FIPS module DB. This allows Firefox to
initialize NSS with the preexisting key and certificate databases available.

MozReview-Commit-ID: 1E4u1ngZyRv

--HG--
rename : security/manager/ssl/tests/unit/test_sdr_preexisting.js => security/manager/ssl/tests/unit/test_broken_fips.js
rename : security/manager/ssl/tests/unit/test_sdr_preexisting/key3.db => security/manager/ssl/tests/unit/test_broken_fips/key3.db
extra : rebase_source : 887f457e998d6e57c6536573fbe3cb10547fe154

security/certverifier/NSSCertDBTrustDomain.cpp

@@ -1227,6 +1227,8 @@ InitializeNSS(const char* dir, bool readOnly, bool loadPKCS11Modules)
   if (!loadPKCS11Modules) {
     flags |= NSS_INIT_NOMODDB;
   }
+  MOZ_LOG(gCertVerifierLog, LogLevel::Debug,
+          ("InitializeNSS(%s, %d, %d)", dir, readOnly, loadPKCS11Modules));
   return ::NSS_Initialize(dir, "", "", SECMOD_DB, flags);
 }
 

security/manager/ssl/nsNSSComponent.cpp

@@ -1635,6 +1635,185 @@ GetNSSProfilePath(nsAutoCString& aProfilePath)
   return NS_OK;
 }
 
+#ifndef ANDROID
+// Given a profile path, attempt to rename the PKCS#11 module DB to
+// "<original name>.fips". In the case of a catastrophic failure (e.g. out of
+// memory), returns a failing nsresult. If execution could conceivably proceed,
+// returns NS_OK even if renaming the file didn't work. This simplifies the
+// logic of the calling code.
+static nsresult
+AttemptToRenamePKCS11ModuleDB(const nsACString& profilePath)
+{
+  // profilePath may come from the environment variable
+  // MOZPSM_NSSDBDIR_OVERRIDE. If so, the user's NSS DBs are most likely not in
+  // their profile directory and we shouldn't mess with them.
+  const char* dbDirOverride = getenv("MOZPSM_NSSDBDIR_OVERRIDE");
+  if (dbDirOverride && strlen(dbDirOverride) > 0) {
+    MOZ_LOG(gPIPNSSLog, LogLevel::Debug,
+            ("MOZPSM_NSSDBDIR_OVERRIDE set - not renaming PKCS#11 module DB"));
+    return NS_OK;
+  }
+  NS_NAMED_LITERAL_CSTRING(moduleDBFilename, "secmod.db");
+  NS_NAMED_LITERAL_CSTRING(destModuleDBFilename, "secmod.db.fips");
+  nsCOMPtr<nsIFile> dbFile = do_CreateInstance("@mozilla.org/file/local;1");
+  if (!dbFile) {
+    return NS_ERROR_FAILURE;
+  }
+  nsresult rv = dbFile->InitWithNativePath(profilePath);
+  if (NS_FAILED(rv)) {
+    return rv;
+  }
+  rv = dbFile->AppendNative(moduleDBFilename);
+  if (NS_FAILED(rv)) {
+    return rv;
+  }
+  // If the PKCS#11 module DB doesn't exist, renaming it won't help.
+  bool exists;
+  rv = dbFile->Exists(&exists);
+  if (NS_FAILED(rv)) {
+    return rv;
+  }
+  // This is strange, but not a catastrophic failure.
+  if (!exists) {
+    MOZ_LOG(gPIPNSSLog, LogLevel::Debug,
+            ("%s doesn't exist?", moduleDBFilename.get()));
+    return NS_OK;
+  }
+  nsCOMPtr<nsIFile> destDBFile = do_CreateInstance("@mozilla.org/file/local;1");
+  if (!destDBFile) {
+    return NS_ERROR_FAILURE;
+  }
+  rv = destDBFile->InitWithNativePath(profilePath);
+  if (NS_FAILED(rv)) {
+    return rv;
+  }
+  rv = destDBFile->AppendNative(destModuleDBFilename);
+  if (NS_FAILED(rv)) {
+    return rv;
+  }
+  // If the destination exists, presumably we've already tried this. Doing it
+  // again won't help.
+  rv = destDBFile->Exists(&exists);
+  if (NS_FAILED(rv)) {
+    return rv;
+  }
+  // Unfortunate, but not a catastrophic failure.
+  if (exists) {
+    MOZ_LOG(gPIPNSSLog, LogLevel::Debug,
+            ("%s already exists - not overwriting",
+             destModuleDBFilename.get()));
+    return NS_OK;
+  }
+  // Now do the actual move.
+  nsCOMPtr<nsIFile> profileDir = do_CreateInstance("@mozilla.org/file/local;1");
+  if (!profileDir) {
+    return NS_ERROR_FAILURE;
+  }
+  rv = profileDir->InitWithNativePath(profilePath);
+  if (NS_FAILED(rv)) {
+    return rv;
+  }
+  // This may fail on, e.g., a read-only file system. This would be unfortunate,
+  // but again it isn't catastropic and we would want to fall back to
+  // initializing NSS in no-DB mode.
+  Unused << dbFile->MoveToNative(profileDir, destModuleDBFilename);
+  return NS_OK;
+}
+#endif // ifndef ANDROID
+
+// Given a profile directory, attempt to initialize NSS. If nocertdb is true,
+// (or if we don't have a profile directory) simply initialize NSS in no DB mode
+// and return. Otherwise, first attempt to initialize in read/write mode, and
+// then read-only mode if that fails. If both attempts fail, we may be failing
+// to initialize an NSS DB collection that has FIPS mode enabled. Attempt to
+// ascertain if this is the case, and if so, rename the offending PKCS#11 module
+// DB so we can (hopefully) initialize NSS in read-write mode. Again attempt
+// read-only mode if that fails. Finally, fall back to no DB mode. On Android
+// we can skip the FIPS workaround since it was never possible to enable FIPS
+// there anyway.
+static nsresult
+InitializeNSSWithFallbacks(const nsACString& profilePath, bool nocertdb,
+                           bool safeMode)
+{
+  if (nocertdb || profilePath.IsEmpty()) {
+    MOZ_LOG(gPIPNSSLog, LogLevel::Debug,
+            ("nocertdb mode or empty profile path -> NSS_NoDB_Init"));
+    SECStatus srv = NSS_NoDB_Init(nullptr);
+    return srv == SECSuccess ? NS_OK : NS_ERROR_FAILURE;
+  }
+
+  const char* profilePathCStr = PromiseFlatCString(profilePath).get();
+  // Try read/write mode. If we're in safeMode, we won't load PKCS#11 modules.
+#ifndef ANDROID
+  PRErrorCode savedPRErrorCode1;
+#endif // ifndef ANDROID
+  SECStatus srv = ::mozilla::psm::InitializeNSS(profilePathCStr, false,
+                                                !safeMode);
+  if (srv == SECSuccess) {
+    MOZ_LOG(gPIPNSSLog, LogLevel::Debug, ("initialized NSS in r/w mode"));
+    return NS_OK;
+  }
+#ifndef ANDROID
+  savedPRErrorCode1 = PR_GetError();
+  PRErrorCode savedPRErrorCode2;
+#endif // ifndef ANDROID
+  // That failed. Try read-only mode.
+  srv = ::mozilla::psm::InitializeNSS(profilePathCStr, true, !safeMode);
+  if (srv == SECSuccess) {
+    MOZ_LOG(gPIPNSSLog, LogLevel::Debug, ("initialized NSS in r-o mode"));
+    return NS_OK;
+  }
+#ifndef ANDROID
+  savedPRErrorCode2 = PR_GetError();
+#endif // ifndef ANDROID
+
+#ifndef ANDROID
+  // That failed as well. Maybe we're trying to load a PKCS#11 module DB that is
+  // in FIPS mode, but we don't support FIPS? Test load NSS without PKCS#11
+  // modules. If that succeeds, that's probably what's going on.
+  if (!safeMode && (savedPRErrorCode1 == SEC_ERROR_LEGACY_DATABASE ||
+                    savedPRErrorCode2 == SEC_ERROR_LEGACY_DATABASE)) {
+    MOZ_LOG(gPIPNSSLog, LogLevel::Debug, ("attempting no-module db init"));
+    // It would make sense to initialize NSS in read-only mode here since this
+    // is just a test to see if the PKCS#11 module DB being in FIPS mode is the
+    // problem, but for some reason the combination of read-only and no-moddb
+    // flags causes NSS initialization to fail, so unfortunately we have to use
+    // read-write mode.
+    srv = ::mozilla::psm::InitializeNSS(profilePathCStr, false, false);
+    if (srv == SECSuccess) {
+      MOZ_LOG(gPIPNSSLog, LogLevel::Debug, ("FIPS may be the problem"));
+      // Unload NSS so we can attempt to fix this situation for the user.
+      srv = NSS_Shutdown();
+      if (srv != SECSuccess) {
+        return NS_ERROR_FAILURE;
+      }
+      MOZ_LOG(gPIPNSSLog, LogLevel::Debug, ("trying to rename module db"));
+      // If this fails non-catastrophically, we'll attempt to initialize NSS
+      // again in r/w then r-o mode (both of which will fail), and then we'll
+      // fall back to NSS_NoDB_Init, which is the behavior we want.
+      nsresult rv = AttemptToRenamePKCS11ModuleDB(profilePath);
+      if (NS_FAILED(rv)) {
+        return rv;
+      }
+      srv = ::mozilla::psm::InitializeNSS(profilePathCStr, false, true);
+      if (srv == SECSuccess) {
+        MOZ_LOG(gPIPNSSLog, LogLevel::Debug, ("initialized in r/w mode"));
+        return NS_OK;
+      }
+      srv = ::mozilla::psm::InitializeNSS(profilePathCStr, true, true);
+      if (srv == SECSuccess) {
+        MOZ_LOG(gPIPNSSLog, LogLevel::Debug, ("initialized in r-o mode"));
+        return NS_OK;
+      }
+    }
+  }
+#endif
+
+  MOZ_LOG(gPIPNSSLog, LogLevel::Debug, ("last-resort NSS_NoDB_Init"));
+  srv = NSS_NoDB_Init(nullptr);
+  return srv == SECSuccess ? NS_OK : NS_ERROR_FAILURE;
+}
+
 nsresult
 nsNSSComponent::InitializeNSS()
 {
@@ -1664,7 +1843,6 @@ nsNSSComponent::InitializeNSS()
     return NS_ERROR_NOT_AVAILABLE;
   }
 
-  SECStatus init_rv = SECFailure;
   bool nocertdb = Preferences::GetBool("security.nocertdb", false);
   bool inSafeMode = true;
   nsCOMPtr<nsIXULRuntime> runtime(do_GetService("@mozilla.org/xre/runtime;1"));
@@ -1679,39 +1857,10 @@ nsNSSComponent::InitializeNSS()
   }
   MOZ_LOG(gPIPNSSLog, LogLevel::Debug, ("inSafeMode: %u\n", inSafeMode));
 
-  PRErrorCode savedPRErrorCode1 = 0;
-  PRErrorCode savedPRErrorCode2 = 0;
-
-  if (!nocertdb && !profileStr.IsEmpty()) {
-    // First try to initialize the NSS DB in read/write mode.
-    // Only load PKCS11 modules if we're not in safe mode.
-    init_rv = ::mozilla::psm::InitializeNSS(profileStr.get(), false, !inSafeMode);
-    // If that fails, attempt read-only mode.
-    if (init_rv != SECSuccess) {
-      savedPRErrorCode1 = PR_GetError();
-      MOZ_LOG(gPIPNSSLog, LogLevel::Debug, ("could not init NSS r/w in %s\n", profileStr.get()));
-      init_rv = ::mozilla::psm::InitializeNSS(profileStr.get(), true, !inSafeMode);
-    }
-    if (init_rv != SECSuccess) {
-      savedPRErrorCode2 = PR_GetError();
-      MOZ_LOG(gPIPNSSLog, LogLevel::Debug, ("could not init in r/o either\n"));
-    }
-  }
-  // If we haven't succeeded in initializing the DB in our profile
-  // directory or we don't have a profile at all, or the "security.nocertdb"
-  // pref has been set to "true", attempt to initialize with no DB.
-  if (nocertdb || init_rv != SECSuccess) {
-    init_rv = NSS_NoDB_Init(nullptr);
-    if (init_rv != SECSuccess) {
-      PRErrorCode savedPRErrorCode3 = PR_GetError();
-      MOZ_CRASH_UNSAFE_PRINTF("NSS initialization failed PRErrorCodes %d %d %d",
-                              savedPRErrorCode1, savedPRErrorCode2,
-                              savedPRErrorCode3);
-    }
-  }
-  if (init_rv != SECSuccess) {
-    MOZ_LOG(gPIPNSSLog, LogLevel::Error, ("could not initialize NSS - panicking\n"));
-    return NS_ERROR_NOT_AVAILABLE;
+  rv = InitializeNSSWithFallbacks(profileStr, nocertdb, inSafeMode);
+  if (NS_FAILED(rv)) {
+    MOZ_LOG(gPIPNSSLog, LogLevel::Debug, ("failed to initialize NSS"));
+    return rv;
   }
 
   // ensure we have an initial value for the content signer root

security/manager/ssl/tests/unit/test_broken_fips.js

@@ -0,0 +1,41 @@
+// -*- indent-tabs-mode: nil; js-indent-level: 2 -*-
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+"use strict";
+
+// Tests that if Firefox attempts and fails to load a PKCS#11 module DB that was
+// in FIPS mode, Firefox can still make use of keys in the key database.
+// secomd.db can be created via `certutil -N -d <dir>`. Putting it in FIPS mode
+// involves running `modutil -fips true -dbdir <dir>`. key3.db is from
+// test_sdr_preexisting/key3.db.
+
+function run_test() {
+  let profile = do_get_profile();
+
+  let keyDBName = "key3.db";
+  let keyDBFile = do_get_file(`test_broken_fips/${keyDBName}`);
+  keyDBFile.copyTo(profile, keyDBName);
+
+  let secmodDBName = "secmod.db";
+  let secmodDBFile = do_get_file(`test_broken_fips/${secmodDBName}`);
+  secmodDBFile.copyTo(profile, secmodDBName);
+
+  let moduleDB = Cc["@mozilla.org/security/pkcs11moduledb;1"]
+                   .getService(Ci.nsIPKCS11ModuleDB);
+  ok(!moduleDB.isFIPSEnabled, "FIPS should not be enabled");
+
+  let sdr = Cc["@mozilla.org/security/sdr;1"]
+              .getService(Ci.nsISecretDecoderRing);
+
+  const encrypted = "MDoEEPgAAAAAAAAAAAAAAAAAAAEwFAYIKoZIhvcNAwcECGeDHwVfyFqzBBAYvqMq/kDMsrARVNdC1C8d";
+  const expectedResult = "password";
+  let decrypted = sdr.decryptString(encrypted);
+  equal(decrypted, expectedResult,
+        "decrypted ciphertext should match expected plaintext");
+
+  let secmodDBFileFIPS = do_get_profile();
+  secmodDBFileFIPS.append(`${secmodDBName}.fips`);
+  ok(secmodDBFileFIPS.exists(), "backed-up PKCS#11 module db should now exist");
+}

security/manager/ssl/tests/unit/xpcshell.ini

@@ -5,6 +5,7 @@ support-files =
   bad_certs/**
   ocsp_certs/**
   test_baseline_requirements/**
+  test_broken_fips/**
   test_cert_eku/**
   test_cert_embedded_null/**
   test_cert_isBuiltInRoot_reload/**
@@ -35,6 +36,11 @@ support-files =
 
 [test_add_preexisting_cert.js]
 [test_baseline_requirements_subject_common_name.js]
+[test_broken_fips.js]
+# FIPS has never been a thing on Android, so the workaround doesn't
+# exist on that platform.
+# FIPS still works on Linux, so this test doesn't make any sense there.
+skip-if = toolkit == 'android' || os == 'linux'
 [test_cert_blocklist.js]
 tags = addons psm blocklist
 [test_cert_chains.js]