Meta+run-tests: Put coverage results in per-library folders

Base/home/anon/Tests/run-tests-and-shutdown.sh

@@ -9,11 +9,8 @@ if [ "$(uname -m)" = "aarch64" -o "$(uname -m)" = "riscv64" ] && [ "$1" != "--fo
     fail_count=0
 }
 else {
-    mkdir -p "$HOME/profiles"
-    export LLVM_PROFILE_FILE="$HOME/profiles/%p-profile.profraw"
     run-tests --show-progress=false --unlink-coredumps
     fail_count=$?
-    unset LLVM_PROFILE_FILE
 }
 
 echo "Failed: $fail_count" > ./test-results.log

Meta/analyze-qemu-coverage.sh

@@ -33,43 +33,38 @@ echo "[ELEVATED] Making sure profile data files are owned by the current user, t
 "$ELEVATE" chown -R "$(id -u)":"$(id -g)" "$TEMP_PROFDATA/"
 
 
-echo "Moving profile data into $TEMP_PROFDATA directly..."
-mv "$TEMP_PROFDATA"/profiles/* "$TEMP_PROFDATA/"
-
 echo "Discovering all binaries and shared libraries in $BUILD_DIR/Root"
 # shellcheck disable=SC2156 # The recommended fix on the Shellcheck github page for this warning causes the script to not find any files at all
-mapfile -d '\n' all_binaries < <(find "$BUILD_DIR"/Root -type f -executable -exec sh -c "file {} | grep -vi relocatable | grep -Eiq ': elf (32|64)-bit'" \; -print | grep -Ev '(usr\/Tests|usr\/local|boot\/|Loader.so)')
-
-# FIXME: Come up with our own coverage prep script instead of using llvm's
-COVERAGE_PREPARE="$BUILD_DIR/prepare-code-coverage-artifact.py"
-if [ ! -f "$COVERAGE_PREPARE" ]; then
-    # Download coverage prep script from github
-    LLVM_GIT_VERSION=9decb102d9a1e7dc55883a633789cb2563de2b25
-    SHA256_SUM=df159c0c9d8129505688ccb42d6066fa36e1bab6e3233417d6c3d26d21d40a5f
-    URL=https://raw.githubusercontent.com/llvm/llvm-project/${LLVM_GIT_VERSION}/llvm/utils/prepare-code-coverage-artifact.py
-
-    echo "Downloading prepare-code-coverage-artifact.py from ${URL}"
-    wget "$URL" -P "$BUILD_DIR"
-
-    # Verify hash matches for integrity
-    echo "Expecting sha256sum: $SHA256_SUM"
-    CALC_SUM="$(sha256sum "${COVERAGE_PREPARE}" | cut -f1 -d' ')"
-    echo "sha256sum($COVERAGE_PREPARE) = '$CALC_SUM'"
-    if [ "$CALC_SUM" != "$SHA256_SUM" ]; then
-        # remove downloaded file to re-download on next run
-        rm -f "$COVERAGE_PREPARE"
-        echo "sha256sums mismatching, removed erroneous download. Please re-try."
-        exit 1
-    fi
-fi
+all_binaries=$(find "$BUILD_DIR"/Root -type f -executable -exec sh -c "file {} | grep -vi relocatable | grep -Eiq ': elf (32|64)-bit'" \; -printf "-object %p\n" | grep -Ev '(usr\/Tests|usr\/local|boot\/|Loader.so)')
 
 CLANG_BINDIR="${SERENITY_ROOT}/Toolchain/Local/clang/bin"
+LLVM_PROFDATA="$CLANG_BINDIR/llvm-profdata"
+PROFDATA_INVOCATION="$LLVM_PROFDATA merge -sparse"
+LLVM_COV="$CLANG_BINDIR/llvm-cov"
 
-# shellcheck disable=SC2128,SC2086 # all_binaries variable needs expanded to space separated string, not newline separated string
-python3 "$COVERAGE_PREPARE" \
-    --unified-report \
-    -C "${SERENITY_ROOT}" \
-    "$CLANG_BINDIR/llvm-profdata" "$CLANG_BINDIR/llvm-cov" \
-    "$TEMP_PROFDATA/" \
-    "$BUILD_DIR/reports/" \
-    $all_binaries
+# Merge profiles
+LIBRARY_DIRS=$(find "$TEMP_PROFDATA/profiles/" -mindepth 1 -type d)
+for dir in $LIBRARY_DIRS;
+do
+    library=$(basename "$dir")
+    echo "Merging profiles for $library"
+    prof_raws=$(find "$dir" -name "*.profraw")
+    # shellcheck disable=SC2086 # The solution would be to use a bash array but it's just easier to omit the quotes.
+    $PROFDATA_INVOCATION $prof_raws -o "$dir/$library.profdata"
+done
+
+echo "Merging all profiles"
+GLOBAL_PROFILE="$TEMP_PROFDATA/global_coverage.profdata"
+PROF_DATA_FILES=$(find "$TEMP_PROFDATA/profiles/" -name "*.profdata")
+# shellcheck disable=SC2086
+$PROFDATA_INVOCATION $PROF_DATA_FILES -o "$GLOBAL_PROFILE"
+
+echo "Generating global html report"
+# shellcheck disable=SC2086
+$LLVM_COV show $all_binaries \
+          -format html \
+          -instr-profile "$GLOBAL_PROFILE" \
+          -o "$BUILD_DIR/reports" \
+          -show-line-counts-or-regions -show-directory-coverage \
+          -Xdemangler "$CLANG_BINDIR/llvm-cxxfilt" -Xdemangler -n \
+          -compilation-dir="${SERENITY_ROOT}"

Userland/Utilities/run-tests.cpp

@@ -7,7 +7,9 @@
 #include <AK/LexicalPath.h>
 #include <LibCore/ArgsParser.h>
 #include <LibCore/ConfigFile.h>
+#include <LibCore/Directory.h>
 #include <LibCore/Environment.h>
+#include <LibCore/StandardPaths.h>
 #include <LibCore/System.h>
 #include <LibCoredump/Backtrace.h>
 #include <LibFileSystem/FileSystem.h>
@@ -269,10 +271,23 @@ FileResult TestRunner::run_test_file(ByteString const& test_path)
     argv.append(nullptr);
 
     pid_t child_pid = -1;
-    // FIXME: Do we really want to copy test runner's entire env?
+    bool is_coverage_run = Core::Environment::has("COVERAGE_RUN"sv);
+    if (is_coverage_run) {
+        // We are modifying the current environment before passing it to the child as it is easier
+        // than recreating one from scratch.
+        auto home = Core::StandardPaths::home_directory();
+        ByteString library_name = path_for_test.parent().basename();
+        auto profile_directory = ByteString::formatted("{}/profiles/{}", home, library_name);
+        MUST(Core::Directory::create(profile_directory, Core::Directory::CreateDirectories::Yes));
+        auto profile_path = ByteString::formatted("{}/%p-profile.profraw", profile_directory);
+        MUST(Core::Environment::set("LLVM_PROFILE_FILE"sv, profile_path, Core::Environment::Overwrite::No));
+    }
+
     int ret = posix_spawn(&child_pid, test_path.characters(), &file_actions, nullptr, const_cast<char* const*>(argv.data()), environ);
     VERIFY(ret == 0);
     VERIFY(child_pid > 0);
+    if (is_coverage_run)
+        MUST(Core::Environment::unset("LLVM_PROFILE_FILE"sv));
 
     int wstatus;