diff --git a/lldb/tools/lldb-perf/darwin/sketch/sketch.cpp b/lldb/tools/lldb-perf/darwin/sketch/sketch.cpp index c8578367b5b0..67f737efae8f 100644 --- a/lldb/tools/lldb-perf/darwin/sketch/sketch.cpp +++ b/lldb/tools/lldb-perf/darwin/sketch/sketch.cpp @@ -204,7 +204,8 @@ public: switch (counter) { case 0: - case 10: + case 10: + case 20: { DoTest (); if (counter == 0) @@ -215,6 +216,7 @@ public: case 1: case 11: + case 21: { DoTest (); m_run_expr_measurement(m_thread.GetFrameAtIndex(0),"properties"); @@ -228,6 +230,7 @@ public: case 2: case 12: + case 22: { DoTest (); next_action.Continue(); @@ -236,6 +239,7 @@ public: case 3: case 13: + case 23: { DoTest (); next_action.StepOver(m_thread); @@ -244,6 +248,7 @@ public: case 4: case 14: + case 24: { DoTest (); @@ -255,6 +260,7 @@ public: case 5: case 15: + case 25: { DoTest (); next_action.StepOver(m_thread); @@ -263,6 +269,7 @@ public: case 6: case 16: + case 26: { DoTest (); next_action.StepOver(m_thread); @@ -271,6 +278,7 @@ public: case 7: case 17: + case 27: { DoTest (); m_run_expr_measurement(m_thread.GetFrameAtIndex(0),"@\"an NSString\""); @@ -282,6 +290,7 @@ public: case 8: case 18: + case 28: { DoTest (); m_run_expr_measurement(m_thread.GetFrameAtIndex(0),"[graphics description]"); @@ -290,6 +299,7 @@ public: } break; case 9: + case 19: { next_action.Relaunch(GetLaunchInfo()); break; @@ -306,11 +316,11 @@ public: virtual void WriteResults (Results &results) { - m_fetch_frames_measurement.WriteAverageValue(results); - m_file_line_bp_measurement.WriteAverageValue(results); - m_fetch_modules_measurement.WriteAverageValue(results); - m_fetch_vars_measurement.WriteAverageValue(results); - m_run_expr_measurement.WriteAverageValue(results); + m_fetch_frames_measurement.WriteAverageAndStandardDeviation(results); + m_file_line_bp_measurement.WriteAverageAndStandardDeviation(results); + m_fetch_modules_measurement.WriteAverageAndStandardDeviation(results); + m_fetch_vars_measurement.WriteAverageAndStandardDeviation(results); + m_run_expr_measurement.WriteAverageAndStandardDeviation(results); results.Write(GetResultFilePath()); } diff --git a/lldb/tools/lldb-perf/lib/Measurement.h b/lldb/tools/lldb-perf/lib/Measurement.h index d43a37962f36..877e618d5469 100644 --- a/lldb/tools/lldb-perf/lib/Measurement.h +++ b/lldb/tools/lldb-perf/lib/Measurement.h @@ -112,6 +112,17 @@ public: results.GetDictionary().Add(metric.GetName(), metric.GetDescription(), lldb_perf::GetResult (NULL, metric.GetAverage())); } + void + WriteAverageAndStandardDeviation (Results &results) + { + auto metric = GetMetric (); + auto dictionary = (Results::Dictionary*)results.GetDictionary().Add(metric.GetName(), metric.GetDescription(), lldb_perf::GetResult (NULL, metric.GetAverage())).get(); + if (dictionary) + { + dictionary->Add("stddev", NULL, lldb_perf::GetResult (NULL, metric.GetStandardDeviation())); + } + } + void WriteStandardDeviation (Results &results) { diff --git a/lldb/tools/lldb-perf/lib/Metric.cpp b/lldb/tools/lldb-perf/lib/Metric.cpp index d8e7935debf8..1951cdb0250a 100644 --- a/lldb/tools/lldb-perf/lib/Metric.cpp +++ b/lldb/tools/lldb-perf/lib/Metric.cpp @@ -57,17 +57,28 @@ Metric::GetAverage () const return GetSum()/GetCount(); } + +// Knuth's algorithm for stddev - massive cancellation resistant template T -Metric::GetStandardDeviation () const +Metric::GetStandardDeviation (StandardDeviationMode mode) const { - T average = GetAverage(); - T diff_squared = 0; - size_t count = GetCount(); - for (auto v : m_dataset) - diff_squared = diff_squared + ( (v-average)*(v-average) ); - diff_squared = diff_squared / count; - return sqrt(diff_squared); + size_t n = 0; + T mean = 0; + T M2 = 0; + for (auto x : m_dataset) + { + n = n + 1; + T delta = x - mean; + mean = mean + delta/n; + M2 = M2+delta*(x-mean); + } + T variance; + if (mode == StandardDeviationMode::ePopulation || n == 1) + variance = M2 / n; + else + variance = M2 / (n - 1); + return sqrt(variance); } template class lldb_perf::Metric; diff --git a/lldb/tools/lldb-perf/lib/Metric.h b/lldb/tools/lldb-perf/lib/Metric.h index 8c32019b21f1..45342d25b41a 100644 --- a/lldb/tools/lldb-perf/lib/Metric.h +++ b/lldb/tools/lldb-perf/lib/Metric.h @@ -22,6 +22,12 @@ template class Metric { public: + enum class StandardDeviationMode + { + eSample, + ePopulation + }; + Metric (); Metric (const char*, const char* = NULL); @@ -38,7 +44,7 @@ public: GetSum () const; ValueType - GetStandardDeviation () const; + GetStandardDeviation (StandardDeviationMode mode = StandardDeviationMode::ePopulation) const; const char* GetName () const diff --git a/lldb/tools/lldb-perf/lib/Results.cpp b/lldb/tools/lldb-perf/lib/Results.cpp index 2bf743598269..6abf67e53b67 100644 --- a/lldb/tools/lldb-perf/lib/Results.cpp +++ b/lldb/tools/lldb-perf/lib/Results.cpp @@ -176,7 +176,7 @@ Results::Write (const char *out_path) #endif } -void +Results::ResultSP Results::Dictionary::AddUnsigned (const char *name, const char *description, uint64_t value) { assert (name && name[0]); @@ -189,9 +189,10 @@ Results::Dictionary::AddUnsigned (const char *name, const char *description, uin } else m_dictionary[std::string(name)] = ResultSP (new Unsigned (name, description, value)); + return m_dictionary[std::string(name)]; } -void +Results::ResultSP Results::Dictionary::AddDouble (const char *name, const char *description, double value) { assert (name && name[0]); @@ -205,8 +206,9 @@ Results::Dictionary::AddDouble (const char *name, const char *description, doubl } else m_dictionary[std::string(name)] = ResultSP (new Double (name, description, value)); + return m_dictionary[std::string(name)]; } -void +Results::ResultSP Results::Dictionary::AddString (const char *name, const char *description, const char *value) { assert (name && name[0]); @@ -219,9 +221,10 @@ Results::Dictionary::AddString (const char *name, const char *description, const } else m_dictionary[std::string(name)] = ResultSP (new String (name, description, value)); + return m_dictionary[std::string(name)]; } -void +Results::ResultSP Results::Dictionary::Add (const char *name, const char *description, const ResultSP &result_sp) { assert (name && name[0]); @@ -234,6 +237,7 @@ Results::Dictionary::Add (const char *name, const char *description, const Resul } else m_dictionary[std::string(name)] = result_sp; + return m_dictionary[std::string(name)]; } void @@ -249,10 +253,11 @@ Results::Dictionary::ForEach (const std::function &callback); - void + ResultSP Add (const char *name, const char *description, const ResultSP &result_sp); - void + ResultSP AddDouble (const char *name, const char *descriptiorn, double value); - void + ResultSP AddUnsigned (const char *name, const char *description, uint64_t value); - void + ResultSP AddString (const char *name, const char *description, const char *value); protected: