Report which modules have forcefully completed types in statistics.

A previous patch added the ability for us to tell if types were forcefully completed. This patch adds the ability to see which modules have forcefully completed types and aggregates the number of modules with forcefully completed types at the root level.

We add a module specific setting named "debugInfoHadIncompleteTypes" that is a boolean value. We also aggregate the number of modules at the root level that had incomplete debug info with a key named "totalModuleCountWithIncompleteTypes" that is a count of number of modules that had incomplete types.

Differential Revision: https://reviews.llvm.org/D138638

lldb/include/lldb/Symbol/TypeSystem.h

@@ -90,7 +90,7 @@ public:
   /// Free up any resources associated with this TypeSystem.  Done before
   /// removing all the TypeSystems from the TypeSystemMap.
   virtual void Finalize() {}
- 
+
   virtual DWARFASTParser *GetDWARFParser() { return nullptr; }
   virtual PDBASTParser *GetPDBParser() { return nullptr; }
   virtual npdb::PdbAstBuilder *GetNativePDBParser() { return nullptr; }
@@ -516,8 +516,13 @@ public:
 
   virtual llvm::Optional<llvm::json::Value> ReportStatistics();
 
+  bool GetHasForcefullyCompletedTypes() const {
+    return m_has_forcefully_completed_types;
+  }
 protected:
   SymbolFile *m_sym_file = nullptr;
+  /// Used for reporting statistics.
+  bool m_has_forcefully_completed_types = false;
 };
 
 class TypeSystemMap {
@@ -541,6 +546,9 @@ public:
   GetTypeSystemForLanguage(lldb::LanguageType language, Target *target,
                            bool can_create);
 
+  /// Check all type systems in the map to see if any have forcefully completed
+  /// types;
+  bool GetHasForcefullyCompletedTypes() const;
 protected:
   typedef llvm::DenseMap<uint16_t, lldb::TypeSystemSP> collection;
   mutable std::mutex m_mutex; ///< A mutex to keep this object happy in

lldb/include/lldb/Target/Statistics.h

@@ -121,6 +121,7 @@ struct ModuleStats {
   bool debug_info_enabled = true;
   bool symtab_stripped = false;
   bool debug_info_had_variable_errors = false;
+  bool debug_info_had_incomplete_types = false;
 };
 
 struct ConstStringStats {

lldb/packages/Python/lldbsuite/test/lldbtest.py

@@ -37,6 +37,7 @@ from functools import wraps
 import gc
 import glob
 import io
+import json
 import os.path
 import re
 import shutil
@@ -1642,6 +1643,19 @@ class Base(unittest2.TestCase):
         err = platform.Run(shell_command)
         return (err, shell_command.GetStatus(), shell_command.GetOutput())
 
+    def get_stats(self, options=None):
+        """
+            Get the output of the "statistics dump" with optional extra options
+            and return the JSON as a python dictionary.
+        """
+        return_obj = lldb.SBCommandReturnObject()
+        command = "statistics dump "
+        if options is not None:
+            command += options
+        self.ci.HandleCommand(command, return_obj, False)
+        metrics_json = return_obj.GetOutput()
+        return json.loads(metrics_json)
+
 # Metaclass for TestBase to change the list of test metods when a new TestCase is loaded.
 # We change the test methods to create a new test method for each test for each debug info we are
 # testing. The name of the new test method will be '<original-name>_<debug-info>' and with adding
@@ -2483,7 +2497,7 @@ FileCheck output:
             error = obj.GetError()
             self.fail(self._formatMessage(msg,
                 "'{}' is not success".format(error)))
-            
+
     """Assert two states are equal"""
     def assertState(self, first, second, msg=None):
         if first != second:

lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp

@@ -226,7 +226,7 @@ static void ForcefullyCompleteType(CompilerType type) {
   auto ts_sp = type.GetTypeSystem();
   auto ts = ts_sp.dyn_cast_or_null<TypeSystemClang>();
   if (ts)
-    ts->GetMetadata(td)->SetIsForcefullyCompleted();
+    ts->SetDeclIsForcefullyCompleted(td);
 }
 
 /// This function serves a similar purpose as RequireCompleteType above, but it

lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp

@@ -9859,7 +9859,7 @@ void TypeSystemClang::RequireCompleteType(CompilerType type) {
   const clang::TagDecl *td = ClangUtil::GetAsTagDecl(type);
   auto ts = type.GetTypeSystem().dyn_cast_or_null<TypeSystemClang>();
   if (ts)
-    ts->GetMetadata(td)->SetIsForcefullyCompleted();
+    ts->SetDeclIsForcefullyCompleted(td);
 }
 
 namespace {
@@ -10062,3 +10062,14 @@ bool TypeSystemClang::IsForcefullyCompleted(lldb::opaque_compiler_type_t type) {
   }
   return false;
 }
+
+bool TypeSystemClang::SetDeclIsForcefullyCompleted(const clang::TagDecl *td) {
+  if (td == nullptr)
+    return false;
+  ClangASTMetadata *metadata = GetMetadata(td);
+  if (metadata == nullptr)
+    return false;
+  m_has_forcefully_completed_types = true;
+  metadata->SetIsForcefullyCompleted();
+  return true;
+}

lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h

@@ -1063,6 +1063,8 @@ public:
   /// complete (base class, member, etc.).
   static void RequireCompleteType(CompilerType type);
 
+  bool SetDeclIsForcefullyCompleted(const clang::TagDecl *td);
+
 private:
   /// Returns the PrintingPolicy used when generating the internal type names.
   /// These type names are mostly used for the formatter selection.

lldb/source/Target/Statistics.cpp

@@ -65,6 +65,8 @@ json::Value ModuleStats::ToJSON() const {
   module.try_emplace("debugInfoEnabled", debug_info_enabled);
   module.try_emplace("debugInfoHadVariableErrors",
                      debug_info_had_variable_errors);
+  module.try_emplace("debugInfoHadIncompleteTypes",
+                     debug_info_had_incomplete_types);
   module.try_emplace("symbolTableStripped", symtab_stripped);
   if (!symfile_path.empty())
     module.try_emplace("symbolFilePath", symfile_path);
@@ -207,6 +209,7 @@ llvm::json::Value DebuggerStats::ReportStatistics(Debugger &debugger,
   uint32_t num_debug_info_enabled_modules = 0;
   uint32_t num_modules_has_debug_info = 0;
   uint32_t num_modules_with_variable_errors = 0;
+  uint32_t num_modules_with_incomplete_types = 0;
   uint32_t num_stripped_modules = 0;
   for (size_t image_idx = 0; image_idx < num_modules; ++image_idx) {
     Module *module = Module::GetAllocatedModuleAtIndex(image_idx);
@@ -273,8 +276,13 @@ llvm::json::Value DebuggerStats::ReportStatistics(Debugger &debugger,
     module->ForEachTypeSystem([&](lldb::TypeSystemSP ts) {
       if (auto stats = ts->ReportStatistics())
         module_stat.type_system_stats.insert({ts->GetPluginName(), *stats});
+      if (ts->GetHasForcefullyCompletedTypes())
+        module_stat.debug_info_had_incomplete_types = true;
       return true;
     });
+    if (module_stat.debug_info_had_incomplete_types)
+      ++num_modules_with_incomplete_types;
+
     json_modules.emplace_back(module_stat.ToJSON());
   }
 
@@ -299,6 +307,7 @@ llvm::json::Value DebuggerStats::ReportStatistics(Debugger &debugger,
       {"totalModuleCount", num_modules},
       {"totalModuleCountHasDebugInfo", num_modules_has_debug_info},
       {"totalModuleCountWithVariableErrors", num_modules_with_variable_errors},
+      {"totalModuleCountWithIncompleteTypes", num_modules_with_incomplete_types},
       {"totalDebugInfoEnabled", num_debug_info_enabled_modules},
       {"totalSymbolTableStripped", num_stripped_modules},
   };

lldb/test/API/commands/statistics/basic/TestStats.py

@@ -55,30 +55,6 @@ class TestCase(TestBase):
         self.assertEqual(success_fail_dict['failures'], num_fails,
                          'make sure success count')
 
-    def get_stats(self, options=None, log_path=None):
-        """
-            Get the output of the "statistics dump" with optional extra options
-            and return the JSON as a python dictionary.
-        """
-        # If log_path is set, open the path and emit the output of the command
-        # for debugging purposes.
-        if log_path is not None:
-            f = open(log_path, 'w')
-        else:
-            f = None
-        return_obj = lldb.SBCommandReturnObject()
-        command = "statistics dump "
-        if options is not None:
-            command += options
-        if f:
-            f.write('(lldb) %s\n' % (command))
-        self.ci.HandleCommand(command, return_obj, False)
-        metrics_json = return_obj.GetOutput()
-        if f:
-            f.write(metrics_json)
-        return json.loads(metrics_json)
-
-
     def get_target_stats(self, debug_stats):
         if "targets" in debug_stats:
             return debug_stats["targets"][0]
@@ -509,7 +485,6 @@ class TestCase(TestBase):
         exe_name = 'a.out'
         exe = self.getBuildArtifact(exe_name)
         dsym = self.getBuildArtifact(exe_name + ".dSYM")
-        print("carp: dsym = '%s'" % (dsym))
         # Make sure the executable file exists after building.
         self.assertEqual(os.path.exists(exe), True)
         # Make sure the dSYM file doesn't exist after building.
@@ -563,7 +538,6 @@ class TestCase(TestBase):
         exe = self.getBuildArtifact(exe_name)
         dsym = self.getBuildArtifact(exe_name + ".dSYM")
         main_obj = self.getBuildArtifact('main.o')
-        print("carp: dsym = '%s'" % (dsym))
         # Make sure the executable file exists after building.
         self.assertEqual(os.path.exists(exe), True)
         # Make sure the dSYM file doesn't exist after building.

lldb/test/API/functionalities/limit-debug-info/TestLimitDebugInfo.py

@@ -30,6 +30,23 @@ class LimitDebugInfoTestCase(TestBase):
         self._check_type(target, "InheritsFromOne")
         self._check_type(target, "InheritsFromTwo")
 
+        # Check that the statistics show that we had incomplete debug info.
+        stats = self.get_stats()
+        # Find the a.out module info in the stats and verify it has the
+        # "debugInfoHadIncompleteTypes" key value pair set to True
+        exe_module_found = False
+        for module in stats['modules']:
+            if module['path'].endswith('a.out'):
+                self.assertTrue(module['debugInfoHadIncompleteTypes'])
+                exe_module_found = True
+                break
+        self.assertTrue(exe_module_found)
+        # Verify that "totalModuleCountWithIncompleteTypes" at the top level
+        # is greater than zero which shows we had incomplete debug info in a
+        # module
+        self.assertGreater(stats['totalModuleCountWithIncompleteTypes'], 0)
+
+
     def _check_incomplete_frame_variable_output(self):
         # Check that the display of the "frame variable" output identifies the
         # incomplete types. Currently the expression parser will find the real