[libFuzzer] extend -rss_limit_mb to crash instantly on a single malloc that exceeds the limit

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@288281 91177308-0d34-0410-b5e6-96231b3b80d8

lib/Fuzzer/FuzzerInternal.h

@@ -105,6 +105,8 @@ public:
   void TryDetectingAMemoryLeak(const uint8_t *Data, size_t Size,
                                bool DuringInitialCorpusExecution);
 
+  void HandleMalloc(size_t Size);
+
 private:
   void AlarmCallback();
   void CrashCallback();

lib/Fuzzer/FuzzerLoop.cpp

@@ -140,6 +140,7 @@ static MallocFreeTracer AllocTracer;
 
 void MallocHook(const volatile void *ptr, size_t size) {
   size_t N = AllocTracer.Mallocs++;
+  F->HandleMalloc(size);
   if (int TraceLevel = AllocTracer.TraceLevel) {
     Printf("MALLOC[%zd] %p %zd\n", N, ptr, size);
     if (TraceLevel >= 2 && EF)
@@ -155,6 +156,21 @@ void FreeHook(const volatile void *ptr) {
   }
 }
 
+// Crash on a single malloc that exceeds the rss limit.
+void Fuzzer::HandleMalloc(size_t Size) {
+  if ((Size >> 20) < (size_t)Options.RssLimitMb)
+    return;
+  Printf("==%d== ERROR: libFuzzer: out-of-memory (malloc(%zd))\n", GetPid(),
+         Size);
+  Printf("   To change the out-of-memory limit use -rss_limit_mb=<N>\n\n");
+  if (EF->__sanitizer_print_stack_trace)
+    EF->__sanitizer_print_stack_trace();
+  DumpCurrentUnit("oom-");
+  Printf("SUMMARY: libFuzzer: out-of-memory\n");
+  PrintFinalStats();
+  _Exit(Options.ErrorExitCode); // Stop right now.
+}
+
 Fuzzer::Fuzzer(UserCallback CB, InputCorpus &Corpus, MutationDispatcher &MD,
                FuzzingOptions Options)
     : CB(CB), Corpus(Corpus), MD(MD), Options(Options) {

lib/Fuzzer/test/CMakeLists.txt

@@ -85,6 +85,7 @@ set(Tests
   NthRunCrashTest
   OneHugeAllocTest
   OutOfMemoryTest
+  OutOfMemorySingleLargeMallocTest
   RepeatedMemcmp
   RepeatedBytesTest
   SimpleCmpTest

lib/Fuzzer/test/OutOfMemorySingleLargeMallocTest.cpp

@@ -0,0 +1,28 @@
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+
+// Tests OOM handling.
+#include <assert.h>
+#include <cstdint>
+#include <cstdlib>
+#include <cstddef>
+#include <cstring>
+#include <iostream>
+#include <unistd.h>
+
+static volatile char *SinkPtr;
+
+extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
+  if (Size > 0 && Data[0] == 'H') {
+    if (Size > 1 && Data[1] == 'i') {
+      if (Size > 2 && Data[2] == '!') {
+          size_t kSize = 0xff000000U;
+          char *p = new char[kSize];
+          SinkPtr = p;
+          delete [] p;
+      }
+    }
+  }
+  return 0;
+}
+

lib/Fuzzer/test/fuzzer-oom-with-profile.test

@@ -1,6 +1,6 @@
 REQUIRES: linux
-RUN: not LLVMFuzzer-OutOfMemoryTest -rss_limit_mb=10 2>&1 | FileCheck %s
+RUN: not LLVMFuzzer-OutOfMemoryTest -rss_limit_mb=300 2>&1 | FileCheck %s
-CHECK: ERROR: libFuzzer: out-of-memory (used: {{.*}}; limit: 10Mb)
+CHECK: ERROR: libFuzzer: out-of-memory (used: {{.*}}; limit: 300Mb)
 CHECK: Live Heap Allocations
 CHECK: Test unit written to ./oom-
 SUMMARY: libFuzzer: out-of-memory

lib/Fuzzer/test/fuzzer-oom.test

@@ -1,4 +1,8 @@
-RUN: not LLVMFuzzer-OutOfMemoryTest -rss_limit_mb=10 2>&1 | FileCheck %s
+RUN: not LLVMFuzzer-OutOfMemoryTest -rss_limit_mb=300 2>&1 | FileCheck %s
-CHECK: ERROR: libFuzzer: out-of-memory (used: {{.*}}; limit: 10Mb)
+CHECK: ERROR: libFuzzer: out-of-memory (used: {{.*}}; limit: 300Mb)
 CHECK: Test unit written to ./oom-
 SUMMARY: libFuzzer: out-of-memory
+
+RUN: not LLVMFuzzer-OutOfMemorySingleLargeMallocTest 2>&1 | FileCheck %s --check-prefix=SINGLE_LARGE_MALLOC
+SINGLE_LARGE_MALLOC: libFuzzer: out-of-memory (malloc(42{{.*}}))
+SINGLE_LARGE_MALLOC: in LLVMFuzzerTestOneInput