ctest_memcheck: Add DEFECT_COUNT option to capture defect count

Auxiliary/vim/syntax/cmake.vim

@@ -128,7 +128,7 @@ syn keyword cmakeKWctest_coverage
             \ contained
 
 syn keyword cmakeKWctest_memcheck
-            \ APPEND BUILD END EXCLUDE EXCLUDE_LABEL INCLUDE INCLUDE_LABEL OFF ON PARALLEL_LEVEL QUIET RETURN_VALUE SCHEDULE_RANDOM START STOP_TIME STRIDE TEST_LOAD
+            \ APPEND BUILD DEFECT_COUNT END EXCLUDE EXCLUDE_LABEL INCLUDE INCLUDE_LABEL OFF ON PARALLEL_LEVEL QUIET RETURN_VALUE SCHEDULE_RANDOM START STOP_TIME STRIDE TEST_LOAD
             \ contained
 
 syn keyword cmakeKWctest_run_script

Help/command/ctest_memcheck.rst

@@ -18,6 +18,7 @@ Perform the :ref:`CTest MemCheck Step` as a :ref:`Dashboard Client`.
                  [SCHEDULE_RANDOM <ON|OFF>]
                  [STOP_TIME <time-of-day>]
                  [RETURN_VALUE <result-var>]
+                 [DEFECT_COUNT <defect-count-var>]
                  [QUIET]
                  )
 
@@ -26,4 +27,9 @@ Run tests with a dynamic analysis tool and store results in
 ``MemCheck.xml`` for submission with the :command:`ctest_submit`
 command.
 
-The options are the same as those for the :command:`ctest_test` command.
+Most options are the same as those for the :command:`ctest_test` command.
+
+The options unique to this command are:
+
+``DEFECT_COUNT <defect-count-var>``
+  Store in the ``<defect-count-var>`` the number of defects found.

Help/release/dev/ctest_memcheck_defect_count.rst

@@ -0,0 +1,5 @@
+ctest_memcheck_defect_count
+---------------------------
+
+* The :command:`ctest_memcheck` command gained a ``DEFECT_COUNT <var>``
+  option to capture the number of memory defects detected.

Source/CTest/cmCTestHandlerCommand.cxx

@@ -226,6 +226,7 @@ bool cmCTestHandlerCommand::InitialPass(std::vector<std::string> const& args,
     this->Makefile->AddDefinition(this->Values[ct_RETURN_VALUE],
                                   str.str().c_str());
   }
+  this->ProcessAdditionalValues(handler);
   // log the error message if there was an error
   if (capureCMakeError) {
     const char* returnString = "0";
@@ -246,6 +247,10 @@ bool cmCTestHandlerCommand::InitialPass(std::vector<std::string> const& args,
   return true;
 }
 
+void cmCTestHandlerCommand::ProcessAdditionalValues(cmCTestGenericHandler*)
+{
+}
+
 bool cmCTestHandlerCommand::CheckArgumentKeyword(std::string const& arg)
 {
   // Look for non-value arguments common to all commands.

Source/CTest/cmCTestHandlerCommand.h

@@ -45,6 +45,8 @@ public:
 protected:
   virtual cmCTestGenericHandler* InitializeHandler() = 0;
 
+  virtual void ProcessAdditionalValues(cmCTestGenericHandler* handler);
+
   // Command argument handling.
   virtual bool CheckArgumentKeyword(std::string const& arg);
   virtual bool CheckArgumentValue(std::string const& arg);

Source/CTest/cmCTestMemCheckCommand.cxx

@@ -4,6 +4,15 @@
 
 #include "cmCTest.h"
 #include "cmCTestGenericHandler.h"
+#include "cmCTestMemCheckHandler.h"
+#include "cmMakefile.h"
+
+cmCTestMemCheckCommand::cmCTestMemCheckCommand()
+{
+  this->Arguments[ctm_DEFECT_COUNT] = "DEFECT_COUNT";
+  this->Arguments[ctm_LAST] = CM_NULLPTR;
+  this->Last = ctm_LAST;
+}
 
 cmCTestGenericHandler* cmCTestMemCheckCommand::InitializeActualHandler()
 {
@@ -28,3 +37,14 @@ cmCTestGenericHandler* cmCTestMemCheckCommand::InitializeActualHandler()
   handler->SetQuiet(this->Quiet);
   return handler;
 }
+
+void cmCTestMemCheckCommand::ProcessAdditionalValues(
+  cmCTestGenericHandler* handler)
+{
+  if (this->Values[ctm_DEFECT_COUNT] && *this->Values[ctm_DEFECT_COUNT]) {
+    std::ostringstream str;
+    str << static_cast<cmCTestMemCheckHandler*>(handler)->GetDefectCount();
+    this->Makefile->AddDefinition(this->Values[ctm_DEFECT_COUNT],
+                                  str.str().c_str());
+  }
+}

Source/CTest/cmCTestMemCheckCommand.h

@@ -20,7 +20,7 @@ class cmCommand;
 class cmCTestMemCheckCommand : public cmCTestTestCommand
 {
 public:
-  cmCTestMemCheckCommand() {}
+  cmCTestMemCheckCommand();
 
   /**
    * This is a virtual constructor for the command.
@@ -40,6 +40,14 @@ public:
 
 protected:
   cmCTestGenericHandler* InitializeActualHandler() CM_OVERRIDE;
+
+  void ProcessAdditionalValues(cmCTestGenericHandler* handler) CM_OVERRIDE;
+
+  enum
+  {
+    ctm_DEFECT_COUNT = ctt_LAST,
+    ctm_LAST
+  };
 };
 
 #endif

Source/CTest/cmCTestMemCheckHandler.cxx

@@ -127,6 +127,7 @@ void cmCTestMemCheckHandler::Initialize()
   this->MemoryTesterOptions.clear();
   this->MemoryTesterStyle = UNKNOWN;
   this->MemoryTesterOutputFile = "";
+  this->DefectCount = 0;
 }
 
 int cmCTestMemCheckHandler::PreProcessHandler()
@@ -279,6 +280,11 @@ void cmCTestMemCheckHandler::PopulateCustomVectors(cmMakefile* mf)
                                      this->Quiet);
 }
 
+int cmCTestMemCheckHandler::GetDefectCount()
+{
+  return this->DefectCount;
+}
+
 void cmCTestMemCheckHandler::GenerateDartOutput(cmXMLWriter& xml)
 {
   if (!this->CTest->GetProduceXML()) {
@@ -702,6 +708,7 @@ bool cmCTestMemCheckHandler::ProcessMemCheckSanitizerOutput(
     ostr << *i << std::endl;
   }
   log = ostr.str();
+  this->DefectCount += defects;
   return defects == 0;
 }
 bool cmCTestMemCheckHandler::ProcessMemCheckPurifyOutput(
@@ -743,6 +750,7 @@ bool cmCTestMemCheckHandler::ProcessMemCheckPurifyOutput(
   }
 
   log = ostr.str();
+  this->DefectCount += defects;
   return defects == 0;
 }
 
@@ -878,6 +886,7 @@ bool cmCTestMemCheckHandler::ProcessMemCheckValgrindOutput(
                        << (cmSystemTools::GetTime() - sttime) << std::endl,
                      this->Quiet);
   log = ostr.str();
+  this->DefectCount += defects;
   return defects == 0;
 }
 
@@ -923,9 +932,9 @@ bool cmCTestMemCheckHandler::ProcessMemCheckBoundsCheckerOutput(
     // only put the output of Bounds Checker if there were
     // errors or leaks detected
     log = parser.Log;
-    return false;
   }
-  return true;
+  this->DefectCount += defects;
+  return defects == 0;
 }
 
 // PostProcessTest memcheck results

Source/CTest/cmCTestMemCheckHandler.h

@@ -30,6 +30,8 @@ public:
 
   void Initialize() CM_OVERRIDE;
 
+  int GetDefectCount();
+
 protected:
   int PreProcessHandler() CM_OVERRIDE;
   int PostProcessHandler() CM_OVERRIDE;
@@ -105,6 +107,7 @@ private:
   std::vector<std::string> ResultStringsLong;
   std::vector<int> GlobalResults;
   bool LogWithPID; // does log file add pid
+  int DefectCount;
 
   std::vector<int>::size_type FindOrAddWarning(const std::string& warning);
   // initialize the ResultStrings and ResultStringsLong for

Tests/RunCMake/ctest_memcheck/DummyLeakSanitizerPrintDefects-result.txt

@@ -0,0 +1 @@
+(-1|255)

Tests/RunCMake/ctest_memcheck/DummyLeakSanitizerPrintDefects-stderr.txt

@@ -0,0 +1 @@
+Defect count: 3

Tests/RunCMake/ctest_memcheck/DummyLeakSanitizerPrintDefects-stdout.txt

@@ -0,0 +1,3 @@
+Memory checking results:
+Direct leak - 2
+Indirect leak - 1

Tests/RunCMake/ctest_memcheck/DummyValgrindNoDefects-result.txt

@@ -0,0 +1 @@
+0

Tests/RunCMake/ctest_memcheck/DummyValgrindNoDefects-stderr.txt

@@ -0,0 +1 @@
+Defect count: 0

Tests/RunCMake/ctest_memcheck/DummyValgrindNoDefects-stdout.txt

@@ -0,0 +1,6 @@
+1/1 MemCheck #1: RunCMake \.+   Passed +[0-9]+.[0-9]+ sec
+
+100% tests passed, 0 tests failed out of 1
+.*
+-- Processing memory checking output:( )
+Memory checking results:

Tests/RunCMake/ctest_memcheck/RunCMakeTest.cmake

@@ -12,6 +12,8 @@ endfunction()
 
 unset(CTEST_EXTRA_CONFIG)
 unset(CTEST_EXTRA_CODE)
+unset(CTEST_SUFFIX_CODE)
+unset(CTEST_MEMCHECK_ARGS)
 unset(CMAKELISTS_EXTRA_CODE)
 
 #-----------------------------------------------------------------------------
@@ -132,4 +134,31 @@ run_mc_test(DummyValgrindNoLogFile "${PSEUDO_VALGRIND_NOLOG}")
 run_mc_test(DummyBCNoLogFile "${PSEUDO_BC_NOLOG}")
 run_mc_test(NotExist "\${CTEST_BINARY_DIRECTORY}/no-memcheck-exe")
 run_mc_test(Unknown "\${CMAKE_COMMAND}")
-run_mc_test(DummyQuiet "${PSEUDO_VALGRIND}" -DMEMCHECK_ARGS=QUIET)
+
+#----------------------------------------------------------------------------
+set(CTEST_MEMCHECK_ARGS QUIET)
+run_mc_test(DummyQuiet "${PSEUDO_VALGRIND}")
+unset(CTEST_MEMCHECK_ARGS)
+
+#-----------------------------------------------------------------------------
+set(CTEST_SUFFIX_CODE "message(\"Defect count: \${defect_count}\")")
+set(CTEST_MEMCHECK_ARGS "DEFECT_COUNT defect_count")
+run_mc_test(DummyValgrindNoDefects "${PSEUDO_VALGRIND}")
+unset(CTEST_MEMCHECK_ARGS)
+unset(CTEST_SUFFIX_CODE)
+
+#-----------------------------------------------------------------------------
+set(CTEST_SUFFIX_CODE "message(\"Defect count: \${defect_count}\")")
+set(CTEST_MEMCHECK_ARGS "DEFECT_COUNT defect_count")
+set(CTEST_EXTRA_CODE
+"set(CTEST_MEMORYCHECK_SANITIZER_OPTIONS \"simulate_sanitizer=1 report_bugs=1 history_size=5 exitcode=55\")
+")
+set(CMAKELISTS_EXTRA_CODE
+"add_test(NAME TestSan COMMAND \"${CMAKE_COMMAND}\"
+-P \"${RunCMake_SOURCE_DIR}/testLeakSanitizer.cmake\")
+")
+run_mc_test(DummyLeakSanitizerPrintDefects "" -DMEMCHECK_TYPE=AddressSanitizer)
+unset(CMAKELISTS_EXTRA_CODE)
+unset(CTEST_EXTRA_CODE)
+unset(CTEST_MEMCHECK_ARGS)
+unset(CTEST_SUFFIX_CODE)

Tests/RunCMake/ctest_memcheck/test.cmake.in

@@ -21,4 +21,6 @@ set(CTEST_MEMORYCHECK_TYPE "${MEMCHECK_TYPE}")
 CTEST_START(Experimental)
 CTEST_CONFIGURE(BUILD "${CTEST_BINARY_DIRECTORY}" RETURN_VALUE res)
 CTEST_TEST(BUILD "${CTEST_BINARY_DIRECTORY}" RETURN_VALUE res)
-CTEST_MEMCHECK(BUILD "${CTEST_BINARY_DIRECTORY}" RETURN_VALUE res ${MEMCHECK_ARGS})
+CTEST_MEMCHECK(BUILD "${CTEST_BINARY_DIRECTORY}" RETURN_VALUE res @CTEST_MEMCHECK_ARGS@)
+
+@CTEST_SUFFIX_CODE@