[ThinLTO]LTO]Legacy] Fix dependent libraries support by adding querying of the IRSymtab

Dependent libraries support for the legacy api was committed in a
broken state (see: https://reviews.llvm.org/D60274). This was missed
due to the painful nature of having to integrate the changes into a
linker in order to test. This change implements support for dependent
libraries in the legacy LTO api:

- I have removed the current api function, which returns a single
string, and   added functions to access each dependent library
specifier individually.

- To reduce the testing pain, I have made the api functions as thin as
possible to   maximize coverage from llvm-lto.

- When doing ThinLTO the system linker will load the modules lazily
when scanning   the input files. Unfortunately, when modules are
lazily loaded there is no access   to module level named metadata. To
fix this I have added api functions that allow   querying the IRSymtab
for the dependent libraries. I hope to expand the api in the   future
so that, eventually, all the information needed by a client linker
during   scan can be retrieved from the IRSymtab.

Differential Revision: https://reviews.llvm.org/D62935

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@363140 91177308-0d34-0410-b5e6-96231b3b80d8

include/llvm-c/lto.h

@@ -297,14 +297,6 @@ lto_module_get_symbol_attribute(lto_module_t mod, unsigned int index);
 extern const char*
 lto_module_get_linkeropts(lto_module_t mod);
 
-/**
-* Returns the module's dependent library specifiers.
-*
-* \since LTO_API_VERSION=24
-*/
-extern const char*
-lto_module_get_dependent_libraries(lto_module_t mod);
-
 /**
  * Diagnostic severity.
  *
@@ -854,7 +846,47 @@ thinlto_codegen_set_cache_size_megabytes(thinlto_code_gen_t cg,
 extern void thinlto_codegen_set_cache_size_files(thinlto_code_gen_t cg,
                                                  unsigned max_size_files);
 
+/** Opaque reference to an LTO input file */
+typedef struct LLVMOpaqueLTOInput *lto_input_t;
 
+/**
+  * Creates an LTO input file from a buffer. The path
+  * argument is used for diagnotics as this function
+  * otherwise does not know which file the given buffer
+  * is associated with.
+  *
+  * \since LTO_API_VERSION=24
+  */
+extern lto_input_t lto_input_create(const void *buffer,
+                                    size_t buffer_size,
+                                    const char *path);
+
+/**
+  * Frees all memory internally allocated by the LTO input file.
+  * Upon return the lto_module_t is no longer valid.
+  *
+  * \since LTO_API_VERSION=24
+  */
+extern void lto_input_dispose(lto_input_t input);
+
+/**
+  * Returns the number of dependent library specifiers
+  * for the given LTO input file.
+  *
+  * \since LTO_API_VERSION=24
+  */
+extern unsigned lto_input_get_num_dependent_libraries(lto_input_t input);
+
+/**
+  * Returns the ith dependent library specifier
+  * for the given LTO input file. The returned
+  * string is not null-terminated.
+  *
+  * \since LTO_API_VERSION=24
+  */
+extern const char * lto_input_get_dependent_library(lto_input_t input,
+                                                    size_t index,
+                                                    size_t *size);
 
 /**
  * @} // endgroup LLVMCTLTO_CACHING

include/llvm/LTO/legacy/LTOModule.h

@@ -17,6 +17,7 @@
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringSet.h"
 #include "llvm/IR/Module.h"
+#include "llvm/LTO/LTO.h"
 #include "llvm/Object/IRObjectFile.h"
 #include "llvm/Object/ModuleSymbolTable.h"
 #include "llvm/Target/TargetMachine.h"
@@ -155,13 +156,19 @@ public:
 
   StringRef getLinkerOpts() { return LinkerOpts; }
 
-  StringRef getDependentLibraries() { return DependentLibraries; }
-
   const std::vector<StringRef> &getAsmUndefinedRefs() { return _asm_undefines; }
 
+  static lto::InputFile *createInputFile(const void *buffer, size_t buffer_size,
+                                         const char *path, std::string &out_error);
+
+  static size_t getDependentLibraryCount(lto::InputFile *input);
+
+  static const char *getDependentLibrary(lto::InputFile *input, size_t index, size_t *size);
+
 private:
   /// Parse metadata from the module
   // FIXME: it only parses "llvm.linker.options" metadata at the moment
+  // FIXME: can't access metadata in lazily loaded modules
   void parseMetadata();
 
   /// Parse the symbols from the module and model-level ASM and add them to

lib/LTO/LTOModule.cpp

@@ -645,10 +645,32 @@ void LTOModule::parseMetadata() {
       continue;
     emitLinkerFlagsForGlobalCOFF(OS, Sym.symbol, TT, M);
   }
-
-  // Dependent Libraries
-  raw_string_ostream OSD(DependentLibraries);
-  if (NamedMDNode *DependentLibraries = getModule().getNamedMetadata("llvm.dependent-libraries"))
-    for (MDNode *N : DependentLibraries->operands())
-      OSD << " " << cast<MDString>(N->getOperand(0))->getString();
+}
+
+lto::InputFile *LTOModule::createInputFile(const void *buffer,
+                                           size_t buffer_size, const char *path,
+                                           std::string &outErr) {
+  StringRef Data((const char *)buffer, buffer_size);
+  MemoryBufferRef BufferRef(Data, path);
+
+  Expected<std::unique_ptr<lto::InputFile>> ObjOrErr =
+      lto::InputFile::create(BufferRef);
+
+  if (ObjOrErr)
+    return ObjOrErr->release();
+
+  outErr = std::string(path) +
+           ": Could not read LTO input file: " + toString(ObjOrErr.takeError());
+  return nullptr;
+}
+
+size_t LTOModule::getDependentLibraryCount(lto::InputFile *input) {
+  return input->getDependentLibraries().size();
+}
+
+const char *LTOModule::getDependentLibrary(lto::InputFile *input, size_t index,
+                                           size_t *size) {
+  StringRef S = input->getDependentLibraries()[index];
+  *size = S.size();
+  return S.data();
 }

test/LTO/X86/Inputs/list-dependent-libraries.ll

@@ -0,0 +1,6 @@
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+!llvm.dependent-libraries = !{!0, !1, !0}
+
+!0 = !{!"foo"}
+!1 = !{!"b a r"}

test/LTO/X86/list-dependent-libraries.ll

@@ -0,0 +1,18 @@
+; RUN: rm -rf %t && mkdir -p %t
+; RUN: llvm-as -o %t/1.bc %s
+; RUN: llvm-as -o %t/2.bc %S/Inputs/list-dependent-libraries.ll
+; RUN: llvm-lto -list-dependent-libraries-only %t/1.bc %t/2.bc | FileCheck %s
+; REQUIRES: default_triple
+
+; CHECK: 1.bc:
+; CHECK-NEXT: wibble
+; CHECK: 2.bc:
+; CHECK-NEXT: foo
+; CHECK-NEXT: b a r
+; CHECK-NEXT: foo
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+!llvm.dependent-libraries = !{!0}
+
+!0 = !{!"wibble"}

test/tools/llvm-lto/error.ll

@@ -4,5 +4,8 @@
 ; RUN: not llvm-lto --list-symbols-only %S/Inputs/empty.bc 2>&1 | FileCheck %s --check-prefix=CHECK-LIST
 ; CHECK-LIST: llvm-lto: error loading file '{{.*}}/Inputs/empty.bc': The file was not recognized as a valid object file
 
+; RUN: not llvm-lto --list-dependent-libraries-only %S/Inputs/empty.bc 2>&1 | FileCheck %s --check-prefix=CHECK-LIBS
+; CHECK-LIBS: llvm-lto: {{.*}}/Inputs/empty.bc: Could not read LTO input file: The file was not recognized as a valid object file
+
 ; RUN: not llvm-lto --thinlto %S/Inputs/empty.bc 2>&1 | FileCheck %s --check-prefix=CHECK-THIN
 ; CHECK-THIN: llvm-lto: error loading file '{{.*}}/Inputs/empty.bc': Invalid bitcode signature

tools/llvm-lto/llvm-lto.cpp

@@ -204,6 +204,10 @@ static cl::opt<bool> ListSymbolsOnly(
     "list-symbols-only", cl::init(false),
     cl::desc("Instead of running LTO, list the symbols in each IR file"));
 
+static cl::opt<bool> ListDependentLibrariesOnly(
+    "list-dependent-libraries-only", cl::init(false),
+    cl::desc("Instead of running LTO, list the dependent libraries in each IR file"));
+
 static cl::opt<bool> SetMergedModule(
     "set-merged-module", cl::init(false),
     cl::desc("Use the first input module as the merged module"));
@@ -372,6 +376,34 @@ static void listSymbols(const TargetOptions &Options) {
   }
 }
 
+static std::unique_ptr<MemoryBuffer> loadFile(StringRef Filename) {
+    ExitOnError ExitOnErr("llvm-lto: error loading file '" + Filename.str() +
+        "': ");
+    return ExitOnErr(errorOrToExpected(MemoryBuffer::getFileOrSTDIN(Filename)));
+}
+
+static void listDependentLibraries() {
+  for (auto &Filename : InputFilenames) {
+    auto Buffer = loadFile(Filename);
+    std::string E;
+    std::unique_ptr<lto::InputFile> Input(LTOModule::createInputFile(
+        Buffer->getBufferStart(), Buffer->getBufferSize(), Filename.c_str(),
+        E));
+    if (!Input)
+      error(E);
+
+    // List the dependent libraries.
+    outs() << Filename << ":\n";
+    for (size_t I = 0, C = LTOModule::getDependentLibraryCount(Input.get());
+         I != C; ++I) {
+      size_t L = 0;
+      const char *S = LTOModule::getDependentLibrary(Input.get(), I, &L);
+      assert(S);
+      outs() << StringRef(S, L) << "\n";
+    }
+  }
+}
+
 /// Create a combined index file from the input IR files and write it.
 ///
 /// This is meant to enable testing of ThinLTO combined index generation,
@@ -449,12 +481,6 @@ std::unique_ptr<ModuleSummaryIndex> loadCombinedIndex() {
   return ExitOnErr(getModuleSummaryIndexForFile(ThinLTOIndex));
 }
 
-static std::unique_ptr<MemoryBuffer> loadFile(StringRef Filename) {
-  ExitOnError ExitOnErr("llvm-lto: error loading file '" + Filename.str() +
-                        "': ");
-  return ExitOnErr(errorOrToExpected(MemoryBuffer::getFileOrSTDIN(Filename)));
-}
-
 static std::unique_ptr<lto::InputFile> loadInputFile(MemoryBufferRef Buffer) {
   ExitOnError ExitOnErr("llvm-lto: error loading input '" +
                         Buffer.getBufferIdentifier().str() + "': ");
@@ -854,6 +880,11 @@ int main(int argc, char **argv) {
     return 0;
   }
 
+  if (ListDependentLibrariesOnly) {
+    listDependentLibraries();
+    return 0;
+  }
+
   if (IndexStats) {
     printIndexStats();
     return 0;

tools/lto/lto.cpp

@@ -18,6 +18,7 @@
 #include "llvm/IR/DiagnosticInfo.h"
 #include "llvm/IR/DiagnosticPrinter.h"
 #include "llvm/IR/LLVMContext.h"
+#include "llvm/LTO/LTO.h"
 #include "llvm/LTO/legacy/LTOCodeGenerator.h"
 #include "llvm/LTO/legacy/LTOModule.h"
 #include "llvm/LTO/legacy/ThinLTOCodeGenerator.h"
@@ -325,10 +326,6 @@ const char* lto_module_get_linkeropts(lto_module_t mod) {
   return unwrap(mod)->getLinkerOpts().data();
 }
 
-const char* lto_module_get_dependent_libraries(lto_module_t mod) {
-    return unwrap(mod)->getDependentLibraries().data();
-}
-
 void lto_codegen_set_diagnostic_handler(lto_code_gen_t cg,
                                         lto_diagnostic_handler_t diag_handler,
                                         void *ctxt) {
@@ -635,3 +632,23 @@ lto_bool_t thinlto_codegen_set_pic_model(thinlto_code_gen_t cg,
   sLastErrorString = "Unknown PIC model";
   return true;
 }
+
+DEFINE_SIMPLE_CONVERSION_FUNCTIONS(lto::InputFile, lto_input_t)
+
+lto_input_t lto_input_create(const void *buffer, size_t buffer_size, const char *path) {
+  return wrap(LTOModule::createInputFile(buffer, buffer_size, path, sLastErrorString));
+}
+
+void lto_input_dispose(lto_input_t input) {
+  delete unwrap(input);
+}
+
+extern unsigned lto_input_get_num_dependent_libraries(lto_input_t input) {
+  return LTOModule::getDependentLibraryCount(unwrap(input));
+}
+
+extern const char *lto_input_get_dependent_library(lto_input_t input,
+                                                   size_t index,
+                                                   size_t *size) {
+  return LTOModule::getDependentLibrary(unwrap(input), index, size);
+}

tools/lto/lto.exports

@@ -72,3 +72,7 @@ thinlto_codegen_disable_codegen
 thinlto_module_get_num_object_files
 thinlto_module_get_object_file
 thinlto_set_generated_objects_dir
+lto_input_create
+lto_input_destroy
+lto_input_get_num_dependent_libraries
+lto_input_get_dependent_library