Bug 1513176: Prevent mozglue.dll from appearing erroneously in the untrusted modules ping;r=aklotz

Nightly is reporting mozglue.dll as being untrusted in the untrusted modules
ping. Until now, xpcshell tests hard-code mozglue to appear as untrusted in
order to cover certain code paths related to startup modules.

This patch:
1. Checks explicitly for the xpcshell environment and only applies this logic
   during xpcshell tests.
2. Uses a purpose-build DLL, "untrusted-startup-test-dll.dll", instead of
   mozglue. This is more explicit and doesn't interfere with any "true"
   processing of mozglue.

Differential Revision: https://phabricator.services.mozilla.com/D14720

--HG--
extra : moz-landing-system : lando

mozglue/build/WindowsDllBlocklist.cpp

@@ -30,6 +30,7 @@
 #include "UntrustedDllsHandler.h"
 #include "nsAutoPtr.h"
 #include "nsWindowsDllInterceptor.h"
+#include "mozilla/CmdLineAndEnvUtils.h"
 #include "mozilla/ScopeExit.h"
 #include "mozilla/Sprintf.h"
 #include "mozilla/StackWalk_windows.h"
@@ -743,6 +744,39 @@ MFBT_API void DllBlocklist_Initialize(uint32_t aInitFlags) {
 #endif
 
   if (IsUntrustedDllsHandlerEnabled()) {
+#ifdef ENABLE_TESTS
+    // Check whether we are running as an xpcshell test.
+    if (mozilla::EnvHasValue("XPCSHELL_TEST_PROFILE_DIR")) {
+      // For xpcshell tests, load this untrusted DLL early enough that the
+      // untrusted module evaluator counts it as a startup module.
+      // It is located in the current directory; the full path must be specified
+      // or LoadLibrary() fails during xpcshell tests with ERROR_MOD_NOT_FOUND.
+
+      // This buffer will hold current directory + dll name
+      wchar_t dllFullPath[MAX_PATH] = {};
+      static const wchar_t kTestDllName[] = L"\\untrusted-startup-test-dll.dll";
+
+      // The amount of the buffer available to store the current directory,
+      // leaving room for the dll name.
+      static const DWORD kBufferDirLen =
+          ArrayLength(dllFullPath) - ArrayLength(kTestDllName);
+
+      DWORD ret = ::GetCurrentDirectoryW(kBufferDirLen, dllFullPath);
+      if ((ret > kBufferDirLen) || !ret) {
+        // Buffer too small or the call failed
+        printf_stderr("Unable to load %S; GetCurrentDirectoryW  failed: %lu",
+                      kTestDllName, GetLastError());
+      } else {
+        wcscat_s(dllFullPath, kTestDllName);
+        HMODULE hTestDll = ::LoadLibraryW(dllFullPath);
+        if (!hTestDll) {
+          printf_stderr("Unable to load %S; LoadLibraryW failed: %lu",
+                        kTestDllName, GetLastError());
+        }
+      }
+    }
+#endif
+
     glue::UntrustedDllsHandler::Init();
   }
 

toolkit/components/telemetry/tests/moz.build

@@ -17,3 +17,8 @@ NO_PGO = True
 if CONFIG['COMPILE_ENVIRONMENT']:
     shared_library = '!%smodules-test%s' % (CONFIG['DLL_PREFIX'], CONFIG['DLL_SUFFIX'])
     TEST_HARNESS_FILES.xpcshell.toolkit.components.telemetry.tests.unit += [shared_library]
+
+if CONFIG['ENABLE_TESTS'] and CONFIG['OS_ARCH'] == 'WINNT':
+    DIRS += [
+        'untrusted-startup-test-dll',
+    ]

toolkit/components/telemetry/tests/unit/test_UntrustedModulesPing.js

@@ -56,7 +56,7 @@ add_task(async function test_send_ping() {
 
     // These check that a DLL loaded at startup is evaluated properly.
     // This is hard-coded as untrusted in ModuleEvaluator for testing.
-    { nameMatch: /mozglue.dll/i, expectedTrusted: false,
+    { nameMatch: /untrusted-startup-test-dll.dll/i, expectedTrusted: false,
         isStartup: true, wasFound: false },
     { nameMatch: /kernelbase.dll/i, expectedTrusted: true,
         isStartup: true, wasFound: false },

toolkit/components/telemetry/tests/untrusted-startup-test-dll/moz.build

@@ -0,0 +1,17 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+DIST_INSTALL = False
+
+SharedLibrary('untrusted-startup-test-dll')
+
+UNIFIED_SOURCES = [
+    'untrusted-startup-test-dll.cpp',
+]
+
+NO_PGO = True
+
+if CONFIG['COMPILE_ENVIRONMENT']:
+    TEST_HARNESS_FILES.xpcshell.toolkit.components.telemetry.tests.unit += ['!untrusted-startup-test-dll.dll']

toolkit/components/telemetry/tests/untrusted-startup-test-dll/untrusted-startup-test-dll.cpp

@@ -0,0 +1,12 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/*
+ * This source file is used to build "untrusted-startup-test-dll.dll" on
+ * Windows. During xpcshell tests, it's loaded early enough to be detected as a
+ * startup module, and hard-coded to be reported in the untrusted modules
+ * telemetry ping, which allows for testing some code paths.
+ */
+
+void nothing() {}

toolkit/xre/ModuleEvaluator_windows.cpp

@@ -10,6 +10,7 @@
 #include <algorithm>  // For std::find()
 
 #include "mozilla/ArrayUtils.h"
+#include "mozilla/CmdLineAndEnvUtils.h"
 #include "mozilla/UniquePtr.h"
 #include "mozilla/Unused.h"
 #include "nsCOMPtr.h"
@@ -200,13 +201,21 @@ Maybe<bool> ModuleEvaluator::IsModuleTrusted(
   }
   ToLowerCase(dllLeafLower);  // To facilitate case-insensitive searching
 
-#if ENABLE_TESTS
+  // Accumulate a trustworthiness score as the module passes through several
+  // checks. If the score ever reaches above the threshold, it's considered
+  // trusted.
   int scoreThreshold = 100;
-  // For testing, these DLLs are hardcoded to pass through all criteria checks
-  // and still result in "untrusted" status.
-  if (dllLeafLower.EqualsLiteral("mozglue.dll") ||
-      dllLeafLower.EqualsLiteral("modules-test.dll")) {
-    scoreThreshold = 99999;
+#ifdef ENABLE_TESTS
+  // Check whether we are running as an xpcshell test.
+  if (mozilla::EnvHasValue("XPCSHELL_TEST_PROFILE_DIR")) {
+    // During xpcshell tests, these DLLs are hard-coded to pass through all
+    // criteria checks and still result in "untrusted" status, so they show up
+    // in the untrusted modules ping for the test to examine.
+    // Setting the threshold very high ensures the test will cover all criteria.
+    if (dllLeafLower.EqualsLiteral("untrusted-startup-test-dll.dll") ||
+        dllLeafLower.EqualsLiteral("modules-test.dll")) {
+      scoreThreshold = 99999;
+    }
   }
 #else
   static const int scoreThreshold = 100;