[ThinLTO] Fix ThinLTOCodegenerator to export llvm.used symbols

Summary:
Reapply r357931 with fixes to ThinLTO testcases and llvm-lto tool.

ThinLTOCodeGenerator currently does not preserve llvm.used symbols and
it can internalize them. In order to pass the necessary information to the
legacy ThinLTOCodeGenerator, the input to the code generator is
rewritten to be based on lto::InputFile.

Now ThinLTO using the legacy LTO API will requires data layout in
Module.

"internalize" thinlto action in llvm-lto is updated to run both
"promote" and "internalize" with the same configuration as
ThinLTOCodeGenerator. The old "promote" + "internalize" option does not
produce the same output as ThinLTOCodeGenerator.

This fixes: PR41236
rdar://problem/49293439

Reviewers: tejohnson, pcc, kromanova, dexonsmith

Reviewed By: tejohnson

Subscribers: ormris, bd1976llvm, mehdi_amini, inglorion, eraman, hiraditya, jkorous, dexonsmith, arphaman, dang, llvm-commits

Tags: #llvm

Differential Revision: https://reviews.llvm.org/D60421

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@358601 91177308-0d34-0410-b5e6-96231b3b80d8

include/llvm/LTO/LTO.h

@@ -131,6 +131,7 @@ public:
     using irsymtab::Symbol::isWeak;
     using irsymtab::Symbol::isIndirect;
     using irsymtab::Symbol::getName;
+    using irsymtab::Symbol::getIRName;
     using irsymtab::Symbol::getVisibility;
     using irsymtab::Symbol::canBeOmittedFromSymbolTable;
     using irsymtab::Symbol::isTLS;
@@ -161,6 +162,9 @@ public:
   // Returns a table with all the comdats used by this file.
   ArrayRef<StringRef> getComdatTable() const { return ComdatTable; }
 
+  // Returns the only BitcodeModule from InputFile.
+  BitcodeModule &getSingleBitcodeModule();
+
 private:
   ArrayRef<Symbol> module_symbols(unsigned I) const {
     const auto &Indices = ModuleSymIndices[I];

include/llvm/LTO/legacy/ThinLTOCodeGenerator.h

@@ -19,6 +19,7 @@
 #include "llvm/ADT/StringSet.h"
 #include "llvm/ADT/Triple.h"
 #include "llvm/IR/ModuleSummaryIndex.h"
+#include "llvm/LTO/LTO.h"
 #include "llvm/Support/CachePruning.h"
 #include "llvm/Support/CodeGen.h"
 #include "llvm/Support/MemoryBuffer.h"
@@ -31,23 +32,6 @@ class StringRef;
 class LLVMContext;
 class TargetMachine;
 
-/// Wrapper around MemoryBufferRef, owning the identifier
-class ThinLTOBuffer {
-  std::string OwnedIdentifier;
-  StringRef Buffer;
-
-public:
-  ThinLTOBuffer(StringRef Buffer, StringRef Identifier)
-      : OwnedIdentifier(Identifier), Buffer(Buffer) {}
-
-  MemoryBufferRef getMemBuffer() const {
-    return MemoryBufferRef(Buffer,
-                           {OwnedIdentifier.c_str(), OwnedIdentifier.size()});
-  }
-  StringRef getBuffer() const { return Buffer; }
-  StringRef getBufferIdentifier() const { return OwnedIdentifier; }
-};
-
 /// Helper to gather options relevant to the target machine creation
 struct TargetMachineBuilder {
   Triple TheTriple;
@@ -267,31 +251,36 @@ public:
    * and additionally resolve weak and linkonce symbols.
    * Index is updated to reflect linkage changes from weak resolution.
    */
-  void promote(Module &Module, ModuleSummaryIndex &Index);
+  void promote(Module &Module, ModuleSummaryIndex &Index,
+               const lto::InputFile &File);
 
   /**
    * Compute and emit the imported files for module at \p ModulePath.
    */
   void emitImports(Module &Module, StringRef OutputName,
-                   ModuleSummaryIndex &Index);
+                   ModuleSummaryIndex &Index,
+                   const lto::InputFile &File);
 
   /**
    * Perform cross-module importing for the module identified by
    * ModuleIdentifier.
    */
-  void crossModuleImport(Module &Module, ModuleSummaryIndex &Index);
+  void crossModuleImport(Module &Module, ModuleSummaryIndex &Index,
+                         const lto::InputFile &File);
 
   /**
    * Compute the list of summaries needed for importing into module.
    */
   void gatherImportedSummariesForModule(
       Module &Module, ModuleSummaryIndex &Index,
-      std::map<std::string, GVSummaryMapTy> &ModuleToSummariesForIndex);
+      std::map<std::string, GVSummaryMapTy> &ModuleToSummariesForIndex,
+      const lto::InputFile &File);
 
   /**
    * Perform internalization. Index is updated to reflect linkage changes.
    */
-  void internalize(Module &Module, ModuleSummaryIndex &Index);
+  void internalize(Module &Module, ModuleSummaryIndex &Index,
+                   const lto::InputFile &File);
 
   /**
    * Perform post-importing ThinLTO optimizations.
@@ -313,7 +302,7 @@ private:
 
   /// Vector holding the input buffers containing the bitcode modules to
   /// process.
-  std::vector<ThinLTOBuffer> Modules;
+  std::vector<std::unique_ptr<lto::InputFile>> Modules;
 
   /// Set of symbols that need to be preserved outside of the set of bitcode
   /// files.

lib/LTO/LTO.cpp

@@ -420,6 +420,11 @@ StringRef InputFile::getName() const {
   return Mods[0].getModuleIdentifier();
 }
 
+BitcodeModule &InputFile::getSingleBitcodeModule() {
+  assert(Mods.size() == 1 && "Expect only one bitcode module");
+  return Mods[0];
+}
+
 LTO::RegularLTOState::RegularLTOState(unsigned ParallelCodeGenParallelismLevel,
                                       Config &Conf)
     : ParallelCodeGenParallelismLevel(ParallelCodeGenParallelismLevel),

lib/LTO/ThinLTOCodeGenerator.cpp

@@ -135,14 +135,13 @@ static void computePrevailingCopies(
   }
 }
 
-static StringMap<MemoryBufferRef>
-generateModuleMap(const std::vector<ThinLTOBuffer> &Modules) {
-  StringMap<MemoryBufferRef> ModuleMap;
-  for (auto &ModuleBuffer : Modules) {
-    assert(ModuleMap.find(ModuleBuffer.getBufferIdentifier()) ==
-               ModuleMap.end() &&
+static StringMap<lto::InputFile *>
+generateModuleMap(std::vector<std::unique_ptr<lto::InputFile>> &Modules) {
+  StringMap<lto::InputFile *> ModuleMap;
+  for (auto &M : Modules) {
+    assert(ModuleMap.find(M->getName()) == ModuleMap.end() &&
            "Expect unique Buffer Identifier");
-    ModuleMap[ModuleBuffer.getBufferIdentifier()] = ModuleBuffer.getMemBuffer();
+    ModuleMap[M->getName()] = M.get();
   }
   return ModuleMap;
 }
@@ -175,18 +174,19 @@ static void verifyLoadedModule(Module &TheModule) {
   }
 }
 
-static std::unique_ptr<Module>
-loadModuleFromBuffer(const MemoryBufferRef &Buffer, LLVMContext &Context,
-                     bool Lazy, bool IsImporting) {
+static std::unique_ptr<Module> loadModuleFromInput(lto::InputFile *Input,
+                                                   LLVMContext &Context,
+                                                   bool Lazy,
+                                                   bool IsImporting) {
+  auto &Mod = Input->getSingleBitcodeModule();
   SMDiagnostic Err;
   Expected<std::unique_ptr<Module>> ModuleOrErr =
-      Lazy
-          ? getLazyBitcodeModule(Buffer, Context,
-                                 /* ShouldLazyLoadMetadata */ true, IsImporting)
-          : parseBitcodeFile(Buffer, Context);
+      Lazy ? Mod.getLazyModule(Context,
+                               /* ShouldLazyLoadMetadata */ true, IsImporting)
+           : Mod.parseModule(Context);
   if (!ModuleOrErr) {
     handleAllErrors(ModuleOrErr.takeError(), [&](ErrorInfoBase &EIB) {
-      SMDiagnostic Err = SMDiagnostic(Buffer.getBufferIdentifier(),
+      SMDiagnostic Err = SMDiagnostic(Mod.getModuleIdentifier(),
                                       SourceMgr::DK_Error, EIB.message());
       Err.print("ThinLTO", errs());
     });
@@ -194,16 +194,17 @@ loadModuleFromBuffer(const MemoryBufferRef &Buffer, LLVMContext &Context,
   }
   if (!Lazy)
     verifyLoadedModule(*ModuleOrErr.get());
-  return std::move(ModuleOrErr.get());
+  return std::move(*ModuleOrErr);
 }
 
 static void
 crossImportIntoModule(Module &TheModule, const ModuleSummaryIndex &Index,
-                      StringMap<MemoryBufferRef> &ModuleMap,
+                      StringMap<lto::InputFile*> &ModuleMap,
                       const FunctionImporter::ImportMapTy &ImportList) {
   auto Loader = [&](StringRef Identifier) {
-    return loadModuleFromBuffer(ModuleMap[Identifier], TheModule.getContext(),
-                                /*Lazy=*/true, /*IsImporting*/ true);
+    auto &Input = ModuleMap[Identifier];
+    return loadModuleFromInput(Input, TheModule.getContext(),
+                               /*Lazy=*/true, /*IsImporting*/ true);
   };
 
   FunctionImporter Importer(Index, Loader);
@@ -248,6 +249,15 @@ static void optimizeModule(Module &TheModule, TargetMachine &TM,
   PM.run(TheModule);
 }
 
+static void
+addUsedSymbolToPreservedGUID(const lto::InputFile &File,
+                             DenseSet<GlobalValue::GUID> &PreservedGUID) {
+  for (const auto &Sym : File.symbols()) {
+    if (Sym.isUsed())
+      PreservedGUID.insert(GlobalValue::getGUID(Sym.getIRName()));
+  }
+}
+
 // Convert the PreservedSymbols map from "Name" based to "GUID" based.
 static DenseSet<GlobalValue::GUID>
 computeGUIDPreservedSymbols(const StringSet<> &PreservedSymbols,
@@ -381,7 +391,7 @@ public:
 
 static std::unique_ptr<MemoryBuffer>
 ProcessThinLTOModule(Module &TheModule, ModuleSummaryIndex &Index,
-                     StringMap<MemoryBufferRef> &ModuleMap, TargetMachine &TM,
+                     StringMap<lto::InputFile *> &ModuleMap, TargetMachine &TM,
                      const FunctionImporter::ImportMapTy &ImportList,
                      const FunctionImporter::ExportSetTy &ExportList,
                      const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols,
@@ -488,15 +498,14 @@ static void initTMBuilder(TargetMachineBuilder &TMBuilder,
 } // end anonymous namespace
 
 void ThinLTOCodeGenerator::addModule(StringRef Identifier, StringRef Data) {
-  ThinLTOBuffer Buffer(Data, Identifier);
-  LLVMContext Context;
-  StringRef TripleStr;
-  ErrorOr<std::string> TripleOrErr = expectedToErrorOrAndEmitErrors(
-      Context, getBitcodeTargetTriple(Buffer.getMemBuffer()));
+  MemoryBufferRef Buffer(Data, Identifier);
 
-  if (TripleOrErr)
-    TripleStr = *TripleOrErr;
+  auto InputOrError = lto::InputFile::create(Buffer);
+  if (!InputOrError)
+    report_fatal_error("ThinLTO cannot create input file: " +
+                       toString(InputOrError.takeError()));
 
+  auto TripleStr = (*InputOrError)->getTargetTriple();
   Triple TheTriple(TripleStr);
 
   if (Modules.empty())
@@ -508,7 +517,7 @@ void ThinLTOCodeGenerator::addModule(StringRef Identifier, StringRef Data) {
     initTMBuilder(TMBuilder, Triple(TMBuilder.TheTriple.merge(TheTriple)));
   }
 
-  Modules.push_back(Buffer);
+  Modules.emplace_back(std::move(*InputOrError));
 }
 
 void ThinLTOCodeGenerator::preserveSymbol(StringRef Name) {
@@ -549,9 +558,10 @@ std::unique_ptr<ModuleSummaryIndex> ThinLTOCodeGenerator::linkCombinedIndex() {
   std::unique_ptr<ModuleSummaryIndex> CombinedIndex =
       llvm::make_unique<ModuleSummaryIndex>(/*HaveGVs=*/false);
   uint64_t NextModuleId = 0;
-  for (auto &ModuleBuffer : Modules) {
-    if (Error Err = readModuleSummaryIndex(ModuleBuffer.getMemBuffer(),
-                                           *CombinedIndex, NextModuleId++)) {
+  for (auto &Mod : Modules) {
+    auto &M = Mod->getSingleBitcodeModule();
+    if (Error Err =
+            M.readSummary(*CombinedIndex, Mod->getName(), NextModuleId++)) {
       // FIXME diagnose
       logAllUnhandledErrors(
           std::move(Err), errs(),
@@ -593,8 +603,8 @@ static void computeDeadSymbolsInIndex(
  * Perform promotion and renaming of exported internal functions.
  * Index is updated to reflect linkage changes from weak resolution.
  */
-void ThinLTOCodeGenerator::promote(Module &TheModule,
-                                   ModuleSummaryIndex &Index) {
+void ThinLTOCodeGenerator::promote(Module &TheModule, ModuleSummaryIndex &Index,
+                                   const lto::InputFile &File) {
   auto ModuleCount = Index.modulePaths().size();
   auto ModuleIdentifier = TheModule.getModuleIdentifier();
 
@@ -606,6 +616,9 @@ void ThinLTOCodeGenerator::promote(Module &TheModule,
   auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(
       PreservedSymbols, Triple(TheModule.getTargetTriple()));
 
+  // Add used symbol to the preserved symbols.
+  addUsedSymbolToPreservedGUID(File, GUIDPreservedSymbols);
+
   // Compute "dead" symbols, we don't want to import/export these!
   computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
 
@@ -633,7 +646,8 @@ void ThinLTOCodeGenerator::promote(Module &TheModule,
  * Perform cross-module importing for the module identified by ModuleIdentifier.
  */
 void ThinLTOCodeGenerator::crossModuleImport(Module &TheModule,
-                                             ModuleSummaryIndex &Index) {
+                                             ModuleSummaryIndex &Index,
+                                             const lto::InputFile &File) {
   auto ModuleMap = generateModuleMap(Modules);
   auto ModuleCount = Index.modulePaths().size();
 
@@ -645,6 +659,8 @@ void ThinLTOCodeGenerator::crossModuleImport(Module &TheModule,
   auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(
       PreservedSymbols, Triple(TheModule.getTargetTriple()));
 
+  addUsedSymbolToPreservedGUID(File, GUIDPreservedSymbols);
+
   // Compute "dead" symbols, we don't want to import/export these!
   computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
 
@@ -663,7 +679,8 @@ void ThinLTOCodeGenerator::crossModuleImport(Module &TheModule,
  */
 void ThinLTOCodeGenerator::gatherImportedSummariesForModule(
     Module &TheModule, ModuleSummaryIndex &Index,
-    std::map<std::string, GVSummaryMapTy> &ModuleToSummariesForIndex) {
+    std::map<std::string, GVSummaryMapTy> &ModuleToSummariesForIndex,
+    const lto::InputFile &File) {
   auto ModuleCount = Index.modulePaths().size();
   auto ModuleIdentifier = TheModule.getModuleIdentifier();
 
@@ -675,6 +692,8 @@ void ThinLTOCodeGenerator::gatherImportedSummariesForModule(
   auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(
       PreservedSymbols, Triple(TheModule.getTargetTriple()));
 
+  addUsedSymbolToPreservedGUID(File, GUIDPreservedSymbols);
+
   // Compute "dead" symbols, we don't want to import/export these!
   computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
 
@@ -693,7 +712,8 @@ void ThinLTOCodeGenerator::gatherImportedSummariesForModule(
  * Emit the list of files needed for importing into module.
  */
 void ThinLTOCodeGenerator::emitImports(Module &TheModule, StringRef OutputName,
-                                       ModuleSummaryIndex &Index) {
+                                       ModuleSummaryIndex &Index,
+                                       const lto::InputFile &File) {
   auto ModuleCount = Index.modulePaths().size();
   auto ModuleIdentifier = TheModule.getModuleIdentifier();
 
@@ -705,6 +725,8 @@ void ThinLTOCodeGenerator::emitImports(Module &TheModule, StringRef OutputName,
   auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(
       PreservedSymbols, Triple(TheModule.getTargetTriple()));
 
+  addUsedSymbolToPreservedGUID(File, GUIDPreservedSymbols);
+
   // Compute "dead" symbols, we don't want to import/export these!
   computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
 
@@ -727,10 +749,12 @@ void ThinLTOCodeGenerator::emitImports(Module &TheModule, StringRef OutputName,
 }
 
 /**
- * Perform internalization. Index is updated to reflect linkage changes.
+ * Perform internalization. Runs promote and internalization together.
+ * Index is updated to reflect linkage changes.
  */
 void ThinLTOCodeGenerator::internalize(Module &TheModule,
-                                       ModuleSummaryIndex &Index) {
+                                       ModuleSummaryIndex &Index,
+                                       const lto::InputFile &File) {
   initTMBuilder(TMBuilder, Triple(TheModule.getTargetTriple()));
   auto ModuleCount = Index.modulePaths().size();
   auto ModuleIdentifier = TheModule.getModuleIdentifier();
@@ -739,6 +763,8 @@ void ThinLTOCodeGenerator::internalize(Module &TheModule,
   auto GUIDPreservedSymbols =
       computeGUIDPreservedSymbols(PreservedSymbols, TMBuilder.TheTriple);
 
+  addUsedSymbolToPreservedGUID(File, GUIDPreservedSymbols);
+
   // Collect for each module the list of function it defines (GUID -> Summary).
   StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
   Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
@@ -758,8 +784,20 @@ void ThinLTOCodeGenerator::internalize(Module &TheModule,
   if (ExportList.empty() && GUIDPreservedSymbols.empty())
     return;
 
-  // Internalization
+  // Resolve prevailing symbols
+  StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
+  resolvePrevailingInIndex(Index, ResolvedODR);
+
+  // Promote the exported values in the index, so that they are promoted
+  // in the module.
   internalizeAndPromoteInIndex(ExportLists, GUIDPreservedSymbols, Index);
+
+  promoteModule(TheModule, Index);
+
+  // Internalization
+  thinLTOResolvePrevailingInModule(
+      TheModule, ModuleToDefinedGVSummaries[ModuleIdentifier]);
+
   thinLTOInternalizeModule(TheModule,
                            ModuleToDefinedGVSummaries[ModuleIdentifier]);
 }
@@ -830,15 +868,14 @@ void ThinLTOCodeGenerator::run() {
     // Perform only parallel codegen and return.
     ThreadPool Pool;
     int count = 0;
-    for (auto &ModuleBuffer : Modules) {
+    for (auto &Mod : Modules) {
       Pool.async([&](int count) {
         LLVMContext Context;
         Context.setDiscardValueNames(LTODiscardValueNames);
 
         // Parse module now
-        auto TheModule =
-            loadModuleFromBuffer(ModuleBuffer.getMemBuffer(), Context, false,
-                                 /*IsImporting*/ false);
+        auto TheModule = loadModuleFromInput(Mod.get(), Context, false,
+                                             /*IsImporting*/ false);
 
         // CodeGen
         auto OutputBuffer = codegenModule(*TheModule, *TMBuilder.create());
@@ -881,6 +918,10 @@ void ThinLTOCodeGenerator::run() {
   auto GUIDPreservedSymbols =
       computeGUIDPreservedSymbols(PreservedSymbols, TMBuilder.TheTriple);
 
+  // Add used symbol from inputs to the preserved symbols.
+  for (const auto &M : Modules)
+    addUsedSymbolToPreservedGUID(*M, GUIDPreservedSymbols);
+
   // Compute "dead" symbols, we don't want to import/export these!
   computeDeadSymbolsInIndex(*Index, GUIDPreservedSymbols);
 
@@ -913,7 +954,7 @@ void ThinLTOCodeGenerator::run() {
   // GVSummary and ResolvedODR maps to enable threaded access to these maps
   // below.
   for (auto &Module : Modules) {
-    auto ModuleIdentifier = Module.getBufferIdentifier();
+    auto ModuleIdentifier = Module->getName();
     ExportLists[ModuleIdentifier];
     ImportLists[ModuleIdentifier];
     ResolvedODR[ModuleIdentifier];
@@ -927,8 +968,10 @@ void ThinLTOCodeGenerator::run() {
   ModulesOrdering.resize(Modules.size());
   std::iota(ModulesOrdering.begin(), ModulesOrdering.end(), 0);
   llvm::sort(ModulesOrdering, [&](int LeftIndex, int RightIndex) {
-    auto LSize = Modules[LeftIndex].getBuffer().size();
-    auto RSize = Modules[RightIndex].getBuffer().size();
+    auto LSize =
+        Modules[LeftIndex]->getSingleBitcodeModule().getBuffer().size();
+    auto RSize =
+        Modules[RightIndex]->getSingleBitcodeModule().getBuffer().size();
     return LSize > RSize;
   });
 
@@ -936,9 +979,9 @@ void ThinLTOCodeGenerator::run() {
   {
     ThreadPool Pool(ThreadCount);
     for (auto IndexCount : ModulesOrdering) {
-      auto &ModuleBuffer = Modules[IndexCount];
+      auto &Mod = Modules[IndexCount];
       Pool.async([&](int count) {
-        auto ModuleIdentifier = ModuleBuffer.getBufferIdentifier();
+        auto ModuleIdentifier = Mod->getName();
         auto &ExportList = ExportLists[ModuleIdentifier];
 
         auto &DefinedGVSummaries = ModuleToDefinedGVSummaries[ModuleIdentifier];
@@ -982,9 +1025,8 @@ void ThinLTOCodeGenerator::run() {
         }
 
         // Parse module now
-        auto TheModule =
-            loadModuleFromBuffer(ModuleBuffer.getMemBuffer(), Context, false,
-                                 /*IsImporting*/ false);
+        auto TheModule = loadModuleFromInput(Mod.get(), Context, false,
+                                             /*IsImporting*/ false);
 
         // Save temps: original file.
         saveTempBitcode(*TheModule, SaveTempsDir, count, ".0.original.bc");

test/Bitcode/Inputs/module_hash.ll

@@ -1,4 +1,6 @@
 ; Needs a function for the combined index to be populated
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
 define void @bar() {
     ret void
 }

test/Bitcode/module_hash.ll

@@ -28,6 +28,7 @@
 ; COMBINED-DAG: <HASH abbrevid={{[0-9]*}} op0=[[HASH1_1]] op1=[[HASH1_2]] op2=[[HASH1_3]] op3=[[HASH1_4]] op4=[[HASH1_5]]/>
 ; COMBINED-DAG: <HASH abbrevid={{[0-9]*}} op0=[[HASH2_1]] op1=[[HASH2_2]] op2=[[HASH2_3]] op3=[[HASH2_4]] op4=[[HASH2_5]]/>
 
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 
 ; Need a function for the combined index to be populated.
 define void @foo() {

test/ThinLTO/X86/Inputs/alias_import.ll

@@ -1,6 +1,4 @@
-
-
-
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 
 @globalfuncAlias = alias void (...), bitcast (void ()* @globalfunc to void (...)*)
 @globalfuncWeakAlias = weak alias void (...), bitcast (void ()* @globalfunc to void (...)*)

test/ThinLTO/X86/Inputs/alias_resolution.ll

@@ -1,6 +1,4 @@
-
-
-
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 
 @globalfuncAlias = alias void (...), bitcast (void ()* @globalfunc to void (...)*)
 @globalfuncWeakAlias = weak alias void (...), bitcast (void ()* @globalfunc to void (...)*)

test/ThinLTO/X86/Inputs/distributed_indexes.ll

@@ -1,3 +1,5 @@
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
 define void @g() {
 entry:
   ret void

test/ThinLTO/X86/Inputs/merge-triple.ll

@@ -1 +1,2 @@
 target triple = "x86_64-apple-macosx10.11.0"
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"

test/ThinLTO/X86/Inputs/section.ll

@@ -1,4 +1,5 @@
 ; An internal global variable that can't be renamed because it has a section
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 @var_with_section = internal global i32 0, section "some_section"
 
 ; @reference_gv_with_section() can't be imported

test/ThinLTO/X86/Inputs/thinlto-internalize-used2.ll

@@ -0,0 +1,10 @@
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.15.0"
+
+define i32 @main() {
+entry:
+  %call = call i32 @bar()
+  ret i32 0
+}
+
+declare i32 @bar()

test/ThinLTO/X86/alias_import.ll

@@ -83,6 +83,8 @@
 ; IMPORT-DAG: declare void @linkonceODRfuncLinkonceAlias()
 ; IMPORT-DAG: define available_externally void @linkonceODRfuncLinkonceODRAlias()
 
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
 define i32 @main() #0 {
 entry:
   call void @globalfuncAlias()

test/ThinLTO/X86/alias_resolution.ll

@@ -43,6 +43,7 @@
 ; PROMOTE_MOD1: @weakfuncLinkonceODRAlias = weak_odr alias void (...), bitcast (void ()* @weakfunc.mod1 to void (...)*)
 ; PROMOTE_MOD2: @weakfuncLinkonceODRAlias = linkonce_odr alias void (...), bitcast (void ()* @weakfunc to void (...)*)
 
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 
 @linkonceODRfuncAlias = alias void (...), bitcast (void ()* @linkonceODRfunc.mod1 to void (...)*)
 @linkonceODRfuncWeakAlias = weak alias void (...), bitcast (void ()* @linkonceODRfunc.mod1 to void (...)*)

test/ThinLTO/X86/deadstrip.ll

@@ -5,8 +5,8 @@
 ; RUN: opt -module-summary %p/Inputs/deadstrip.ll -o %t2.bc
 ; RUN: llvm-lto -thinlto-action=thinlink -o %t.index.bc %t1.bc %t2.bc
 
-; RUN: llvm-lto -exported-symbol=_main -thinlto-action=promote %t1.bc -thinlto-index=%t.index.bc -o - | llvm-lto -exported-symbol=_main -thinlto-action=internalize -thinlto-index %t.index.bc -thinlto-module-id=%t1.bc - -o - | llvm-dis -o - | FileCheck %s
-; RUN: llvm-lto -exported-symbol=_main -thinlto-action=promote %t2.bc -thinlto-index=%t.index.bc -o - | llvm-lto -exported-symbol=_main -thinlto-action=internalize -thinlto-index %t.index.bc -thinlto-module-id=%t2.bc - -o - | llvm-dis -o - | FileCheck %s --check-prefix=CHECK2
+; RUN: llvm-lto -exported-symbol=_main -thinlto-action=internalize %t1.bc -thinlto-index=%t.index.bc -o - | llvm-dis -o - | FileCheck %s
+; RUN: llvm-lto -exported-symbol=_main -thinlto-action=internalize %t2.bc -thinlto-index=%t.index.bc -o - | llvm-dis -o - | FileCheck %s --check-prefix=CHECK2
 
 ; RUN: llvm-lto -exported-symbol=_main -thinlto-action=run -stats %t1.bc %t2.bc 2>&1 | FileCheck %s --check-prefix=STATS
 ; RUN: llvm-nm %t1.bc.thinlto.o | FileCheck %s --check-prefix=CHECK-NM

test/ThinLTO/X86/distributed_indexes.ll

@@ -46,6 +46,8 @@
 ; RUN: llvm-dis %t1.bc.thinlto.bc -o - | FileCheck %s --check-prefix=DIS
 ; DIS: aliasee: null
 
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
 declare void @g(...)
 declare void @analias(...)
 

test/ThinLTO/X86/internalize.ll

@@ -33,7 +33,7 @@
 ; INTERNALIZE: define internal void @linkonce_func()
 ; INTERNALIZE-OPTION-DISABLE: define void @foo
 ; INTERNALIZE-OPTION-DISABLE: define void @bar
-; INTERNALIZE-OPTION-DISABLE: define linkonce void @linkonce_func()
+; INTERNALIZE-OPTION-DISABLE: define weak void @linkonce_func()
 ; INTERNALIZE2: define dso_local void @foo
 ; INTERNALIZE2: define internal void @bar
 ; INTERNALIZE2: define internal void @linkonce_func()

test/ThinLTO/X86/linkonce_odr_unnamed_addr.ll

@@ -6,5 +6,7 @@
 ; RUN: llvm-lto -thinlto-action=thinlink -o %t3.bc %t.bc %t2.bc
 ; RUN: llvm-lto -thinlto-action=promote %t.bc -thinlto-index=%t3.bc -o - | llvm-dis -o - | FileCheck %s
 
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
 ; CHECK: @linkonceodrunnamed = weak_odr hidden unnamed_addr constant i32 0
 @linkonceodrunnamed = linkonce_odr unnamed_addr constant i32 0

test/ThinLTO/X86/merge-triple.ll

@@ -4,6 +4,7 @@
 ; RUN: llvm-dis < %t1.bc.thinlto.imported.bc | FileCheck %s --check-prefix=CHECK1
 ; RUN: llvm-dis < %t2.bc.thinlto.imported.bc | FileCheck %s --check-prefix=CHECK2
 
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.12.0"
 
 ; CHECK1: target triple = "x86_64-apple-macosx10.12.0"

test/ThinLTO/X86/prefix_replace.ll

@@ -10,6 +10,7 @@
 ; RUN: llvm-lto -thinlto-action=distributedindexes -thinlto-prefix-replace="%t/oldpath/;%t/newpath/" -thinlto-index %t.index.bc %t/oldpath/prefix_replace.o
 
 ; RUN: ls %t/newpath/prefix_replace.o.thinlto.bc
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 
 define void @f() {
 entry:

test/ThinLTO/X86/section.ll

@@ -12,6 +12,7 @@
 ; IMPORT: declare void @reference_gv_with_section()
 ; Canary to check that importing is correctly set up.
 ; IMPORT: define available_externally void @foo()
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 
 
 define i32 @main() {

test/ThinLTO/X86/thinlto-internalize-used.ll

@@ -0,0 +1,21 @@
+; RUN: opt -module-summary -o %t.bc %s
+; RUN: opt -module-summary -o %t-main.bc %S/Inputs/thinlto-internalize-used2.ll
+; RUN: llvm-lto -thinlto-action=thinlink %t.bc %t-main.bc -o %t-index.bc
+; RUN: llvm-lto -thinlto-action=internalize -thinlto-index %t-index.bc %t.bc -o %t.promote.bc
+; RUN: llvm-dis %t.promote.bc -o - | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.15.0"
+
+@llvm.used = appending global [1 x i8*] [i8* bitcast (i32 ()* @foo to i8*)], section "llvm.metadata"
+
+; Make sure foo is not internalized.
+; CHECK: define i32 @foo()
+define i32 @foo() {
+  ret i32 0
+}
+
+define hidden i32 @bar() {
+  ret i32 0
+}
+

test/ThinLTO/X86/weak_resolution.ll

@@ -7,7 +7,7 @@
 ; non-prevailing ODR are not kept when possible, but non-ODR non-prevailing
 ; are not affected.
 ; RUN: llvm-lto -thinlto-action=promote %t.bc -thinlto-index=%t3.bc -o - | llvm-dis -o - | FileCheck %s --check-prefix=MOD1
-; RUN: llvm-lto -thinlto-action=promote %t.bc -thinlto-index=%t3.bc -exported-symbol=linkoncefunc -o - | llvm-lto -thinlto-action=internalize -thinlto-module-id=%t.bc - -thinlto-index=%t3.bc -exported-symbol=linkoncefunc -o - | llvm-dis -o - | FileCheck %s --check-prefix=MOD1-INT
+; RUN: llvm-lto -thinlto-action=internalize %t.bc -thinlto-index=%t3.bc -exported-symbol=linkoncefunc -o - | llvm-dis -o - | FileCheck %s --check-prefix=MOD1-INT
 ; RUN: llvm-lto -thinlto-action=promote %t2.bc -thinlto-index=%t3.bc -o - | llvm-dis -o - | FileCheck %s --check-prefix=MOD2
 ; When exported, we always preserve a linkonce
 ; RUN: llvm-lto -thinlto-action=promote %t.bc -thinlto-index=%t3.bc -o - --exported-symbol=linkonceodrfuncInSingleModule | llvm-dis -o - | FileCheck %s --check-prefix=EXPORTED

test/ThinLTO/X86/weak_resolution_single.ll

@@ -1,9 +1,10 @@
 ; RUN: opt -module-summary %s -o %t.bc
 ; RUN: llvm-lto -thinlto-action=thinlink -o %t2.bc %t.bc
 
-; RUN: llvm-lto -thinlto-action=promote %t.bc -thinlto-index=%t2.bc -exported-symbol=foo -o - | llvm-lto -thinlto-action=internalize -thinlto-module-id=%t.bc - -thinlto-index=%t2.bc -exported-symbol=foo -o - | llvm-dis -o - | FileCheck %s
+; RUN: llvm-lto -thinlto-action=internalize %t.bc -thinlto-index=%t2.bc -exported-symbol=foo -o - | llvm-dis -o - | FileCheck %s
 
 ; CHECK: define weak_odr void @foo()
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 define linkonce_odr void @foo() {
   ret void
 }

test/Transforms/ThinLTOBitcodeWriter/no-type-md.ll

@@ -27,6 +27,8 @@
 
 ; BCA: <GLOBALVAL_SUMMARY_BLOCK
 
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
 ; CHECK: @g = global i8 42
 @g = global i8 42
 

tools/llvm-lto/llvm-lto.cpp

@@ -449,22 +449,37 @@ std::unique_ptr<ModuleSummaryIndex> loadCombinedIndex() {
   return ExitOnErr(getModuleSummaryIndexForFile(ThinLTOIndex));
 }
 
-static std::unique_ptr<Module> loadModule(StringRef Filename,
-                                          LLVMContext &Ctx) {
-  SMDiagnostic Err;
-  std::unique_ptr<Module> M(parseIRFile(Filename, Err, Ctx));
-  if (!M) {
-    Err.print("llvm-lto", errs());
-    report_fatal_error("Can't load module for file " + Filename);
-  }
-  maybeVerifyModule(*M);
+static std::unique_ptr<MemoryBuffer> loadFile(StringRef Filename) {
+  ExitOnError ExitOnErr("llvm-lto: error loading file '" + Filename.str() +
+                        "': ");
+  return ExitOnErr(errorOrToExpected(MemoryBuffer::getFileOrSTDIN(Filename)));
+}
 
+static std::unique_ptr<lto::InputFile> loadInputFile(MemoryBufferRef Buffer) {
+  ExitOnError ExitOnErr("llvm-lto: error loading input '" +
+                        Buffer.getBufferIdentifier().str() + "': ");
+  return ExitOnErr(lto::InputFile::create(Buffer));
+}
+
+static std::unique_ptr<Module> loadModuleFromInput(lto::InputFile &File,
+                                                   LLVMContext &CTX) {
+  auto &Mod = File.getSingleBitcodeModule();
+  auto ModuleOrErr = Mod.parseModule(CTX);
+  if (!ModuleOrErr) {
+    handleAllErrors(ModuleOrErr.takeError(), [&](ErrorInfoBase &EIB) {
+      SMDiagnostic Err = SMDiagnostic(Mod.getModuleIdentifier(),
+                                      SourceMgr::DK_Error, EIB.message());
+      Err.print("llvm-lto", errs());
+    });
+    report_fatal_error("Can't load module, abort.");
+  }
+  maybeVerifyModule(**ModuleOrErr);
   if (ThinLTOModuleId.getNumOccurrences()) {
     if (InputFilenames.size() != 1)
       report_fatal_error("Can't override the module id for multiple files");
-    M->setModuleIdentifier(ThinLTOModuleId);
+    (*ModuleOrErr)->setModuleIdentifier(ThinLTOModuleId);
   }
-  return M;
+  return std::move(*ModuleOrErr);
 }
 
 static void writeModuleToFile(Module &TheModule, StringRef Filename) {
@@ -562,13 +577,15 @@ private:
     auto Index = loadCombinedIndex();
     for (auto &Filename : InputFilenames) {
       LLVMContext Ctx;
-      auto TheModule = loadModule(Filename, Ctx);
+      auto Buffer = loadFile(Filename);
+      auto Input = loadInputFile(Buffer->getMemBufferRef());
+      auto TheModule = loadModuleFromInput(*Input, Ctx);
 
       // Build a map of module to the GUIDs and summary objects that should
       // be written to its index.
       std::map<std::string, GVSummaryMapTy> ModuleToSummariesForIndex;
-      ThinGenerator.gatherImportedSummariesForModule(*TheModule, *Index,
-                                                     ModuleToSummariesForIndex);
+      ThinGenerator.gatherImportedSummariesForModule(
+          *TheModule, *Index, ModuleToSummariesForIndex, *Input);
 
       std::string OutputName = OutputFilename;
       if (OutputName.empty()) {
@@ -597,13 +614,16 @@ private:
     auto Index = loadCombinedIndex();
     for (auto &Filename : InputFilenames) {
       LLVMContext Ctx;
-      auto TheModule = loadModule(Filename, Ctx);
+      auto Buffer = loadFile(Filename);
+      auto Input = loadInputFile(Buffer->getMemBufferRef());
+      auto TheModule = loadModuleFromInput(*Input, Ctx);
       std::string OutputName = OutputFilename;
       if (OutputName.empty()) {
         OutputName = Filename + ".imports";
       }
-      OutputName = getThinLTOOutputFile(OutputName, OldPrefix, NewPrefix);
-      ThinGenerator.emitImports(*TheModule, OutputName, *Index);
+      OutputName =
+          getThinLTOOutputFile(OutputName, OldPrefix, NewPrefix);
+      ThinGenerator.emitImports(*TheModule, OutputName, *Index, *Input);
     }
   }
 
@@ -621,9 +641,11 @@ private:
     auto Index = loadCombinedIndex();
     for (auto &Filename : InputFilenames) {
       LLVMContext Ctx;
-      auto TheModule = loadModule(Filename, Ctx);
+      auto Buffer = loadFile(Filename);
+      auto Input = loadInputFile(Buffer->getMemBufferRef());
+      auto TheModule = loadModuleFromInput(*Input, Ctx);
 
-      ThinGenerator.promote(*TheModule, *Index);
+      ThinGenerator.promote(*TheModule, *Index, *Input);
 
       std::string OutputName = OutputFilename;
       if (OutputName.empty()) {
@@ -652,9 +674,11 @@ private:
 
     for (auto &Filename : InputFilenames) {
       LLVMContext Ctx;
-      auto TheModule = loadModule(Filename, Ctx);
+      auto Buffer = loadFile(Filename);
+      auto Input = loadInputFile(Buffer->getMemBufferRef());
+      auto TheModule = loadModuleFromInput(*Input, Ctx);
 
-      ThinGenerator.crossModuleImport(*TheModule, *Index);
+      ThinGenerator.crossModuleImport(*TheModule, *Index, *Input);
 
       std::string OutputName = OutputFilename;
       if (OutputName.empty()) {
@@ -683,9 +707,11 @@ private:
 
     for (auto &Filename : InputFilenames) {
       LLVMContext Ctx;
-      auto TheModule = loadModule(Filename, Ctx);
+      auto Buffer = loadFile(Filename);
+      auto Input = loadInputFile(Buffer->getMemBufferRef());
+      auto TheModule = loadModuleFromInput(*Input, Ctx);
 
-      ThinGenerator.internalize(*TheModule, *Index);
+      ThinGenerator.internalize(*TheModule, *Index, *Input);
 
       std::string OutputName = OutputFilename;
       if (OutputName.empty()) {
@@ -706,7 +732,9 @@ private:
 
     for (auto &Filename : InputFilenames) {
       LLVMContext Ctx;
-      auto TheModule = loadModule(Filename, Ctx);
+      auto Buffer = loadFile(Filename);
+      auto Input = loadInputFile(Buffer->getMemBufferRef());
+      auto TheModule = loadModuleFromInput(*Input, Ctx);
 
       ThinGenerator.optimize(*TheModule);