Backed out changeset a1b7ab570480 (bug 1305970) for bustage

security/nss/Makefile

@@ -27,7 +27,7 @@ include $(CORE_DEPTH)/coreconf/config.mk
 #######################################################################
 
 ifdef NSS_DISABLE_GTESTS
-DIRS := $(filter-out gtests,$(DIRS))
+DIRS := $(filter-out external_tests,$(DIRS))
 endif
 
 #######################################################################

security/nss/TAG-INFO

@@ -1 +1 @@
-b9bf17853004
+3e7b53b18112

security/nss/automation/taskcluster/graph/src/extend.js

@@ -14,10 +14,14 @@ const WINDOWS_CHECKOUT_CMD =
 
 /*****************************************************************************/
 
+function isSanitizer(task) {
+  return task.collection == "asan" || task.collection == "ubsan";
+}
+
 queue.filter(task => {
   if (task.group == "Builds") {
     // Remove extra builds on {A,UB}San and ARM.
-    if (task.collection == "asan" || task.collection == "arm-debug") {
+    if (isSanitizer(task) || task.collection == "arm-debug") {
       return false;
     }
 
@@ -26,6 +30,11 @@ queue.filter(task => {
         (task.platform != "linux64" || task.collection != "debug")) {
       return false;
     }
+
+    // Remove extra builds on gyp builds (TODO: add when it supports CC/CCC).
+    if (task.collection == "gyp") {
+      return false;
+    }
   }
 
   if (task.tests == "bogo") {
@@ -40,11 +49,17 @@ queue.filter(task => {
     }
   }
 
+  // Filter test suites that currently fail with UBSan.
+  if (task.collection == "ubsan" &&
+      ["crmf", "cipher", "fips", "merge", "smime"].includes(task.tests)) {
+    return false;
+  }
+
   return true;
 });
 
 queue.map(task => {
-  if (task.collection == "asan") {
+  if (isSanitizer(task)) {
     // CRMF and FIPS tests still leak, unfortunately.
     if (task.tests == "crmf" || task.tests == "fips") {
       task.env.ASAN_OPTIONS = "detect_leaks=0";
@@ -104,13 +119,27 @@ export default async function main() {
       "-c",
       "bin/checkout.sh && nss/automation/taskcluster/scripts/build_gyp.sh"
     ],
-    env: {USE_64: "1"}, // This is only necessary for tests to work.
+    env: {USE_64: "1"},
     platform: "linux64",
     collection: "gyp",
     image: LINUX_IMAGE
   });
 
   await scheduleLinux("Linux 64 (ASan, debug)", {
+    env: {
+      NSS_DISABLE_ARENA_FREE_LIST: "1",
+      NSS_DISABLE_UNLOAD: "1",
+      CC: "clang",
+      CCC: "clang++",
+      USE_ASAN: "1",
+      USE_64: "1"
+    },
+    platform: "linux64",
+    collection: "asan",
+    image: LINUX_IMAGE
+  });
+
+  await scheduleLinux("Linux 64 (ASan+UBSan, debug)", {
     env: {
       UBSAN_OPTIONS: "print_stacktrace=1",
       NSS_DISABLE_ARENA_FREE_LIST: "1",
@@ -122,7 +151,7 @@ export default async function main() {
       USE_64: "1"
     },
     platform: "linux64",
-    collection: "asan",
+    collection: "ubsan",
     image: LINUX_IMAGE
   });
 
@@ -134,8 +163,6 @@ export default async function main() {
     collection: "debug"
   });
 
-  await scheduleFuzzing();
-
   await scheduleTools();
 
   await scheduleLinux("Linux 32 (ARM, debug)", {
@@ -234,65 +261,6 @@ async function scheduleLinux(name, base) {
 
 /*****************************************************************************/
 
-async function scheduleFuzzing() {
-  let base = {
-    env: {
-      ASAN_OPTIONS: "allocator_may_return_null=1",
-      UBSAN_OPTIONS: "print_stacktrace=1",
-      CC: "clang",
-      CCC: "clang++",
-      USE_64: "1" // This is only necessary for tests to work.
-    },
-    platform: "linux64",
-    collection: "fuzz",
-    image: LINUX_IMAGE
-  };
-
-  // Build base definition.
-  let build_base = merge({
-    command: [
-      "/bin/bash",
-      "-c",
-      "bin/checkout.sh && " +
-      "nss/automation/taskcluster/scripts/build_gyp.sh -g -v --fuzz"
-    ],
-    artifacts: {
-      public: {
-        expires: 24 * 7,
-        type: "directory",
-        path: "/home/worker/artifacts"
-      }
-    },
-    kind: "build",
-    symbol: "B"
-  }, base);
-
-  // The task that builds NSPR+NSS.
-  let task_build = queue.scheduleTask(merge(build_base, {
-    name: "Linux x64 (debug, fuzz)"
-  }));
-
-  // Schedule tests.
-  queue.scheduleTask(merge(base, {
-    parent: task_build,
-    name: "Gtests",
-    command: [
-      "/bin/bash",
-      "-c",
-      "bin/checkout.sh && nss/automation/taskcluster/scripts/run_tests.sh"
-    ],
-    env: {GTESTFILTER: "*Fuzz*"},
-    symbol: "Gtest",
-    tests: "gtests",
-    cycle: "standard",
-    kind: "test"
-  }));
-
-  return queue.submit();
-}
-
-/*****************************************************************************/
-
 async function scheduleWindows(name, base) {
   base = merge(base, {
     workerType: "nss-win2012r2",

security/nss/automation/taskcluster/graph/src/try_syntax.js

@@ -22,8 +22,7 @@ function parseOptions(opts) {
   }
 
   // Parse platforms.
-  let allPlatforms = ["linux", "linux64", "linux64-asan", "win64", "arm",
-                      "linux64-gyp", "linux64-fuzz"];
+  let allPlatforms = ["linux", "linux64", "linux64-asan", "win64", "arm", "linux64-gyp"];
   let platforms = intersect(opts.platform.split(/\s*,\s*/), allPlatforms);
 
   // If the given value is nonsense or "none" default to all platforms.
@@ -102,7 +101,6 @@ function filter(opts) {
       let aliases = {
         "linux": "linux32",
         "linux64-asan": "linux64",
-        "linux64-fuzz": "linux64",
         "linux64-gyp": "linux64",
         "win64": "windows2012-64",
         "arm": "linux32"
@@ -113,13 +111,11 @@ function filter(opts) {
 
       // Additional checks.
       if (platform == "linux64-asan") {
-        keep &= coll("asan");
+        keep &= coll("asan") || coll("ubsan");
       } else if (platform == "arm") {
         keep &= coll("arm-opt") || coll("arm-debug");
       } else if (platform == "linux64-gyp") {
         keep &= coll("gyp");
-      } else if (platform == "linux64-fuzz") {
-        keep &= coll("fuzz");
       } else {
         keep &= coll("opt") || coll("debug");
       }
@@ -132,8 +128,7 @@ function filter(opts) {
     }
 
     // Finally, filter by build type.
-    let isDebug = coll("debug") || coll("asan") || coll("arm-debug") ||
-                  coll("gyp") || coll("fuzz");
+    let isDebug = coll("debug") || coll("asan") || coll("ubsan") || coll("arm-debug") || coll("gyp");
     return (isDebug && opts.builds.includes("d")) ||
            (!isDebug && opts.builds.includes("o"));
   }

security/nss/automation/taskcluster/scripts/build_gyp.sh

@@ -4,14 +4,14 @@ source $(dirname $0)/tools.sh
 
 if [[ $(id -u) -eq 0 ]]; then
     # Drop privileges by re-running this script.
-    exec su worker -c "$0 $*"
+    exec su worker $0
 fi
 
 # Clone NSPR if needed.
 hg_clone https://hg.mozilla.org/projects/nspr nspr default
 
 # Build.
-nss/build.sh ${*--g -v}
+nss/build.sh -g -v
 
 # Package.
 mkdir artifacts

security/nss/automation/taskcluster/scripts/run_clang_format.sh

@@ -19,7 +19,6 @@ else
     top=$(dirname $0)/../../..
     dirs=( \
          "$top/cmd" \
-         "$top/fuzz" \
          "$top/lib/base" \
          "$top/lib/certdb" \
          "$top/lib/certhigh" \
@@ -29,22 +28,16 @@ else
          "$top/lib/dbm" \
          "$top/lib/dev" \
          "$top/lib/freebl" \
-         "$top/lib/jar" \
          "$top/lib/nss" \
-         "$top/lib/pk11wrap" \
-         "$top/lib/pkcs7" \
-         "$top/lib/pkcs12" \
-         "$top/lib/pki" \
-         "$top/lib/smime" \
          "$top/lib/softoken" \
          "$top/lib/ssl" \
-         "$top/lib/sysinit" \
          "$top/lib/util" \
-         "$top/gtests/common" \
-         "$top/gtests/der_gtest" \
-         "$top/gtests/pk11_gtest" \
-         "$top/gtests/ssl_gtest" \
-         "$top/gtests/util_gtest" \
+         "$top/lib/sysinit" \
+         "$top/external_tests/common" \
+         "$top/external_tests/der_gtest" \
+         "$top/external_tests/pk11_gtest" \
+         "$top/external_tests/ssl_gtest" \
+         "$top/external_tests/util_gtest" \
     )
 fi
 

security/nss/automation/taskcluster/scripts/run_scan_build.sh

@@ -23,13 +23,8 @@ declare -A scan=( \
         [lib/base]=0 \
         [lib/certdb]=0 \
         [lib/certhigh]=0 \
-        [lib/ckfw]=0 \
-        [lib/crmf]=0 \
-        [lib/cryptohi]=0 \
-        [lib/dev]=0 \
-        [lib/freebl]=0 \
-        [lib/nss]=0 \
         [lib/ssl]=0 \
+        [lib/freebl]=0 \
         [lib/util]=0 \
     )
 

security/nss/build.sh

@@ -3,129 +3,70 @@
 #
 # This build system is still under development.  It does not yet support all
 # the features or platforms that NSS supports.
+#
+# -c = clean before build
+# -g = force a rebuild of gyp (and NSPR, because why not)
+# -v = verbose build
+# --test = ignore map files and export everything we have
 
 set -e
 
-# Usage info
-show_help() {
-cat << EOF
-
-Usage: ${0##*/} [-hcgv] [-j <n>] [--test] [--fuzz] [--scan-build[=output]]
-                [-m32] [--opt|-o]
-
-This script builds NSS with gyp and ninja.
-
-This build system is still under development.  It does not yet support all
-the features or platforms that NSS supports.
-
-NSS build tool options:
-
-    -h            display this help and exit
-    -c            clean before build
-    -g            force a rebuild of gyp (and NSPR, because why not)
-    -j <n>        run at most <n> concurrent jobs
-    -v            verbose build
-    -m32          do a 32-bit build on a 64-bit system
-    --test        ignore map files and export everything we have
-    --fuzz        enable fuzzing mode. this always enables test builds
-    --scan-build  run the build with scan-build (scan-build has to be in the path)
-                  --scan-build=/out/path sets the output path for scan-build
-    --opt|-o      do an opt build
-EOF
-}
+CWD=$(cd $(dirname $0); pwd -P)
+OBJ_DIR=$(make -s -C "$CWD" platform)
+DIST_DIR="$CWD/../dist/$OBJ_DIR"
 
 if [ -n "$CCC" ] && [ -z "$CXX" ]; then
     export CXX="$CCC"
 fi
 
-opt_build=0
-build_64=0
-clean=0
-rebuild_gyp=0
-target=Debug
-
-# try to guess sensible defaults
-arch=$(uname -m)
-if [ "$arch" = "x86_64" -o "$arch" = "aarch64" ]; then
-    build_64=1
-fi
-
-gyp_params=()
-ninja_params=()
-scanbuild=()
-nspr_env=()
-
-# parse command line arguments
 while [ $# -gt 0 ]; do
     case $1 in
-        -c) clean=1 ;;
-        -g) rebuild_gyp=1 ;;
-        -j) ninja_params+=(-j "$2"); shift ;;
-        -v) ninja_params+=(-v) ;;
-        --test) gyp_params+=(-Dtest_build=1) ;;
-        --fuzz) gyp_params+=(-Dtest_build=1 -Dfuzz=1) ;;
-        --scan-build) scanbuild=(scan-build) ;;
-        --scan-build=?*) scanbuild=(scan-build -o "${1#*=}") ;;
-        --opt|-o) opt_build=1; nspr_env+=(BUILD_OPT=1) ;;
-        -m32|--m32) build_64=0 ;;
-        *) show_help; exit ;;
+        -c) CLEAN=1 ;;
+        -g) REBUILD_GYP=1 ;;
+        -v) VERBOSE=1 ;;
+        --test) GYP_PARAMS="$GYP_PARAMS -Dtest_build=1" ;;
     esac
     shift
 done
 
-# set paths
-cwd=$(cd $(dirname $0); pwd -P)
-obj_dir=$(USE_64=$build_64 make -s -C "$cwd" platform)
-dist_dir="$cwd/../dist/$obj_dir"
-
 # -c = clean first
-if [ "$clean" = 1 ]; then
-    rm -rf "$cwd/out"
-    rm -rf "$cwd/../nspr/$obj_dir"
+if [ "$CLEAN" = 1 ]; then
+    rm -rf "$CWD/out"
 fi
 
-if [ "$opt_build" = "1" ]; then
-    target=Release
+if [ "$BUILD_OPT" = "1" ]; then
+    TARGET=Release
 else
-    target=Debug
+    TARGET=Debug
 fi
-if [ "$build_64" == "1" ]; then
-    target="${target}_x64"
-    nspr_env+=(USE_64=1)
+if [ "$USE_64" == "1" ]; then
+    TARGET="${TARGET}_x64"
 else
-    gyp_params+=(-Dtarget_arch=ia32)
+    GYP_PARAMS="$GYP_PARAMS -Dtarget_arch=ia32"
 fi
-target_dir="$cwd/out/$target"
-
-# figure out the scan-build string
-if [ "${#scanbuild[@]}" -gt 0 ]; then
-    if [ -n "$CC" ]; then
-       scanbuild+=(--use-cc="$CC")
-    fi
-    if [ -n "$CCC" ]; then
-       scanbuild+=(--use-c++="$CCC")
-    fi
- fi
+TARGET_DIR="$CWD/out/$TARGET"
 
 # These steps can take a while, so don't overdo them.
 # Force a redo with -g.
-if [ "$rebuild_gyp" = 1 -o ! -d "$target_dir" ]; then
+if [ "$REBUILD_GYP" = 1 -o ! -d "$TARGET_DIR" ]; then
     # Build NSPR.
-    make "${nspr_env[@]}" -C "$cwd" NSS_GYP=1 install_nspr
+    make -C "$CWD" NSS_GYP=1 install_nspr
 
     # Run gyp.
-    PKG_CONFIG_PATH="$cwd/../nspr/$obj_dir/config" \
-        "${scanbuild[@]}" gyp -f ninja "${gyp_params[@]}" --depth="$cwd" \
-          --generator-output="." "$cwd/nss.gyp"
+    PKG_CONFIG_PATH="$CWD/../nspr/$OBJ_DIR/config" $SCANBUILD \
+        gyp -f ninja $GYP_PARAMS --depth="$CWD" --generator-output="." "$CWD/nss.gyp"
 fi
 
 # Run ninja.
 if which ninja >/dev/null 2>&1; then
-    ninja=(ninja)
+    NINJA=ninja
 elif which ninja-build >/dev/null 2>&1; then
-    ninja=(ninja-build)
+    NINJA=ninja-build
 else
     echo "Please install ninja" 1>&2
     exit 1
 fi
-"${scanbuild[@]}" $ninja -C "$target_dir" "${ninja_params[@]}"
+if [ "$VERBOSE" = 1 ]; then
+    NINJA="$NINJA -v"
+fi
+$NINJA -C "$TARGET_DIR"

security/nss/cmd/lib/basicutil.c

@@ -792,7 +792,6 @@ parseGroupList(const char *arg, SSLNamedGroup **enabledGroups,
         p = strtok(NULL, ",");
     }
     PORT_Free(str);
-    str = NULL;
     groups = PORT_ZNewArray(SSLNamedGroup, numValues);
     if (!groups) {
         goto done;

security/nss/cmd/tstclnt/tstclnt.c

@@ -1471,7 +1471,7 @@ main(int argc, char **argv)
     if (requireDHNamedGroups) {
         rv = SSL_OptionSet(s, SSL_REQUIRE_DH_NAMED_GROUPS, PR_TRUE);
         if (rv != SECSuccess) {
-            SECU_PrintError(progName, "error in requiring the use of fixed finite-field DH groups");
+            SECU_PrintError(progName, "error enabling extended master secret");
             error = 1;
             goto done;
         }

security/nss/coreconf/config.gypi

@@ -52,11 +52,6 @@
           'dll_prefix': 'lib',
           'dll_suffix': 'so',
         }],
-        ['OS=="linux"', {
-          'freebl_name': 'freeblpriv3',
-        }, {
-          'freebl_name': 'freebl3',
-        }],
         ['OS=="mac"', {
           'zlib_libs%': ['-lz'],
           'use_system_sqlite%': 1,
@@ -90,7 +85,6 @@
     'sqlite_libs%': ['-lsqlite3'],
     'dll_prefix': '<(dll_prefix)',
     'dll_suffix': '<(dll_suffix)',
-    'freebl_name': '<(freebl_name)',
     'cc_is_clang%': '<(cc_is_clang)',
     # Some defaults
     'disable_tests%': 0,
@@ -104,7 +98,6 @@
     'ssl_enable_zlib%': 1,
     'use_asan%': 0,
     'test_build%': 0,
-    'fuzz%': 0,
   },
   'target_defaults': {
     # Settings specific to targets should go here.
@@ -286,9 +279,6 @@
               '<!@(<(python) <(DEPTH)/coreconf/werror.py)',
             ],
           }],
-          [ 'fuzz==1', {
-            'cflags': ['-Wno-unused-function']
-          }],
           [ 'OS=="android" and mozilla_client==0', {
             'defines': [
               'NO_SYSINFO',

security/nss/coreconf/coreconf.dep

@@ -10,4 +10,3 @@
  */
 
 #error "Do not include this header file."
-

security/nss/external_tests/common/common.gyp

@@ -19,15 +19,15 @@
         '<(DEPTH)/lib/util/util.gyp:nssutil3',
         '<(DEPTH)/lib/smime/smime.gyp:smime3',
         '<(DEPTH)/lib/ssl/ssl.gyp:ssl3',
-        '<(DEPTH)/gtests/google_test/google_test.gyp:gtest',
+        '<(DEPTH)/external_tests/google_test/google_test.gyp:gtest',
         '<(DEPTH)/cmd/lib/lib.gyp:sectool'
       ]
     }
   ],
   'target_defaults': {
     'include_dirs': [
-      '../../gtests/google_test/gtest/include',
-      '../../gtests/common'
+      '../../external_tests/google_test/gtest/include',
+      '../../external_tests/common'
     ],
   },
   'variables': {

security/nss/external_tests/common/gtest.gypi

@@ -17,11 +17,6 @@
           '-lstdc++',
         ],
       }],
-      [ 'fuzz==1', {
-        'defines': [
-          'UNSAFE_FUZZER_MODE',
-        ],
-      }],
     ],
     'msvs_settings': {
       'VCCLCompilerTool': {

security/nss/external_tests/common/manifest.mn

@@ -10,12 +10,12 @@ CPPSRCS = \
       gtests.cc \
       $(NULL)
 
-INCLUDES += -I$(CORE_DEPTH)/gtests/google_test/gtest/include \
-            -I$(CORE_DEPTH)/gtests/common
+INCLUDES += -I$(CORE_DEPTH)/external_tests/google_test/gtest/include \
+            -I$(CORE_DEPTH)/external_tests/common
 
 REQUIRES = gtest
 
 EXTRA_LIBS = $(DIST)/lib/$(LIB_PREFIX)gtest.$(LIB_SUFFIX)
 
 # NOTE: this is not actually used but required to build gtests.o
-PROGRAM = gtests
+PROGRAM = gtests

security/nss/external_tests/der_gtest/der_gtest.gyp

@@ -12,18 +12,23 @@
       'type': 'executable',
       'sources': [
         'der_getint_unittest.cc',
-        '<(DEPTH)/gtests/common/gtests.cc'
+        '<(DEPTH)/external_tests/common/gtests.cc'
       ],
       'dependencies': [
         '<(DEPTH)/exports.gyp:nss_exports',
-        '<(DEPTH)/gtests/google_test/google_test.gyp:gtest',
+        '<(DEPTH)/lib/nss/nss.gyp:nss3',
+        '<(DEPTH)/lib/util/util.gyp:nssutil3',
+        '<(DEPTH)/lib/smime/smime.gyp:smime3',
+        '<(DEPTH)/lib/ssl/ssl.gyp:ssl3',
+        '<(DEPTH)/external_tests/google_test/google_test.gyp:gtest',
+        '<(DEPTH)/cmd/lib/lib.gyp:sectool'
       ]
     }
   ],
   'target_defaults': {
     'include_dirs': [
-      '../../gtests/google_test/gtest/include',
-      '../../gtests/common'
+      '../../external_tests/google_test/gtest/include',
+      '../../external_tests/common'
     ]
   },
   'variables': {

security/nss/external_tests/der_gtest/manifest.mn

@@ -11,8 +11,8 @@ CPPSRCS = \
       der_private_key_import_unittest.cc \
       $(NULL)
 
-INCLUDES += -I$(CORE_DEPTH)/gtests/google_test/gtest/include \
-            -I$(CORE_DEPTH)/gtests/common
+INCLUDES += -I$(CORE_DEPTH)/external_tests/google_test/gtest/include \
+            -I$(CORE_DEPTH)/external_tests/common
 
 REQUIRES = nspr nss libdbm gtest