[lld-link] Add /reproduce: support for several flags

/reproduce: now works correctly with:
- /call-graph-ordering-file:
- /def:
- /natvis:
- /order:
- /pdbstream:

I went through all instances of MemoryBuffer::getFile() and made sure
everything that didn't already do so called takeBuffer().

For natvis, that wasn't possible since DebugInfo/PDB wants to take
owernship of the natvis buffer. For that case, I'm manually adding the
tar file entry.

/natvis: and /pdbstream: is slightly awkward, since createResponseFile()
always adds these flags to the response file but createPDB() (which
ultimately adds the files referenced by the flags) is only called if
/debug is also passed. So when using /natvis: without /debug with
/reproduce:, lld won't warn, but when linking using the response
file from the archive, it won't find the natvis file since it's not
in the tar. This isn't a new issue though, and after this patch things
at least work with using /natvis: _with_ debug with /reproduce:.
(Same for /pdbstream:)

Differential Revison: https://reviews.llvm.org/D97212

lld/COFF/Driver.cpp

@@ -54,7 +54,7 @@
 using namespace llvm;
 using namespace llvm::object;
 using namespace llvm::COFF;
-using llvm::sys::Process;
+using namespace llvm::sys;
 
 namespace lld {
 namespace coff {
@@ -622,6 +622,14 @@ static uint64_t getDefaultImageBase() {
   return config->dll ? 0x10000000 : 0x400000;
 }
 
+static std::string rewritePath(StringRef s) {
+  if (fs::exists(s))
+    return relativeToRoot(s);
+  return std::string(s);
+}
+
+// Reconstructs command line arguments so that so that you can re-run
+// the same command with the same inputs. This is for --reproduce.
 static std::string createResponseFile(const opt::InputArgList &args,
                                       ArrayRef<StringRef> filePaths,
                                       ArrayRef<StringRef> searchPaths) {
@@ -642,6 +650,24 @@ static std::string createResponseFile(const opt::InputArgList &args,
     case OPT_manifestinput:
     case OPT_manifestuac:
       break;
+    case OPT_call_graph_ordering_file:
+    case OPT_deffile:
+    case OPT_natvis:
+      os << arg->getSpelling() << quote(rewritePath(arg->getValue())) << '\n';
+      break;
+    case OPT_order: {
+      StringRef orderFile = arg->getValue();
+      orderFile.consume_front("@");
+      os << arg->getSpelling() << '@' << quote(rewritePath(orderFile)) << '\n';
+      break;
+    }
+    case OPT_pdbstream: {
+      const std::pair<StringRef, StringRef> nameFile =
+          StringRef(arg->getValue()).split("=");
+      os << arg->getSpelling() << nameFile.first << '='
+         << quote(rewritePath(nameFile.second)) << '\n';
+      break;
+    }
     case OPT_implib:
     case OPT_pdb:
     case OPT_pdbstripped:
@@ -838,6 +864,9 @@ static void parseModuleDefs(StringRef path) {
   COFFModuleDefinition m = check(parseCOFFModuleDefinition(
       mb->getMemBufferRef(), config->machine, config->mingw));
 
+  // Include in /reproduce: output if applicable.
+  driver->takeBuffer(std::move(mb));
+
   if (config->outputFile.empty())
     config->outputFile = std::string(saver.save(m.OutputFile));
   config->importName = std::string(saver.save(m.ImportName));
@@ -938,6 +967,9 @@ static void parseOrderFile(StringRef arg) {
     else
       config->order[s] = INT_MIN + config->order.size();
   }
+
+  // Include in /reproduce: output if applicable.
+  driver->takeBuffer(std::move(mb));
 }
 
 static void parseCallGraphFile(StringRef path) {
@@ -978,6 +1010,9 @@ static void parseCallGraphFile(StringRef path) {
       if (SectionChunk *to = findSection(fields[1]))
         config->callGraphProfile[{from, to}] += count;
   }
+
+  // Include in /reproduce: output if applicable.
+  driver->takeBuffer(std::move(mb));
 }
 
 static void readCallGraphsFromObjectFiles() {

lld/COFF/Driver.h

@@ -93,9 +93,9 @@ public:
 
   void enqueuePath(StringRef path, bool wholeArchive, bool lazy);
 
-private:
   std::unique_ptr<llvm::TarWriter> tar; // for /linkrepro
 
+private:
   // Searches a file from search paths.
   Optional<StringRef> findFile(StringRef filename);
   Optional<StringRef> findLib(StringRef filename);

lld/COFF/PDB.cpp

@@ -1093,7 +1093,14 @@ void PDBLinker::addNatvisFiles() {
       warn("Cannot open input file: " + file);
       continue;
     }
-    builder.addInjectedSource(file, std::move(*dataOrErr));
+    std::unique_ptr<MemoryBuffer> data = std::move(*dataOrErr);
+
+    // Can't use takeBuffer() here since addInjectedSource() takes ownership.
+    if (driver->tar)
+      driver->tar->append(relativeToRoot(data->getBufferIdentifier()),
+                          data->getBuffer());
+
+    builder.addInjectedSource(file, std::move(data));
   }
 }
 
@@ -1106,7 +1113,9 @@ void PDBLinker::addNamedStreams() {
       warn("Cannot open input file: " + file);
       continue;
     }
-    exitOnErr(builder.addNamedStream(stream, (*dataOrErr)->getBuffer()));
+    std::unique_ptr<MemoryBuffer> data = std::move(*dataOrErr);
+    exitOnErr(builder.addNamedStream(stream, data->getBuffer()));
+    driver->takeBuffer(std::move(data));
   }
 }
 

lld/test/COFF/linkrepro-manifest.test

@@ -1,7 +1,13 @@
 REQUIRES: x86, gnutar, manifest_tool
 
+manifest-related files are compiled to a .res file and the .res file is
+added to the repro archive, instead of adding the inputs.
+
 RUN: rm -rf %t && mkdir %t && cd %t
-RUN: lld-link -entry:__ImageBase -nodefaultlib -linkrepro:%t -manifest:embed %p/Inputs/std32.lib -subsystem:console
+RUN: lld-link -entry:__ImageBase -nodefaultlib -linkrepro:%t \
+RUN:     -manifest:embed %p/Inputs/std32.lib -subsystem:console \
+RUN:     -manifestinput:%p/Inputs/manifestinput.test
+
 RUN: tar tf repro.tar | FileCheck --check-prefix=LIST %s
 RUN: tar xOf repro.tar repro/response.txt | FileCheck %s
 

lld/test/COFF/linkrepro-pdb.test

@@ -4,19 +4,31 @@ RUN: rm -rf %t && mkdir -p %t && cd %t
 RUN: yaml2obj %S/Inputs/pdb-type-server-simple-a.yaml -o a.obj
 RUN: yaml2obj %S/Inputs/pdb-type-server-simple-b.yaml -o b.obj
 RUN: llvm-pdbutil yaml2pdb %S/Inputs/pdb-type-server-simple-ts.yaml -pdb ts.pdb
-RUN: lld-link a.obj b.obj -entry:main -debug -out:%t.exe -pdb:%t.pdb -nodefaultlib -linkrepro:.
-RUN: tar xOf repro.tar repro/%:t/ts.pdb > repro-ts.pdb
-RUN: diff ts.pdb repro-ts.pdb
-
+RUN: cp %p/Inputs/natvis-1.natvis %t.natvis
+RUN: cp %p/Inputs/stream.txt %t.txt
+RUN: lld-link a.obj b.obj -entry:main -debug -out:%t.exe -pdb:%t.pdb \
+RUN:     -nodefaultlib -linkrepro:. -natvis:%t.natvis \
+RUN:     -pdbstream:srcsrv=%t.txt
+RUN: tar tf repro.tar | FileCheck --check-prefix=LIST %s
 RUN: tar xf repro.tar
-RUN: cat repro/response.txt | FileCheck -check-prefix=PDB %s
+RUN: diff ts.pdb repro/%:t/ts.pdb
+RUN: diff %t.natvis repro/%:t.natvis
+RUN: diff %t.txt repro/%:t.txt
+RUN: cat repro/response.txt | FileCheck -check-prefix=RSP %s
 
-PDB: -out:linkrepro-pdb.test.tmp.exe
-PDB-NEXT: -pdb:linkrepro-pdb.test.tmp.pdb
+LIST: .obj
+LIST: response.txt
+LIST: .natvis
+
+RSP: -out:linkrepro-pdb.test.tmp.exe
+RSP-NEXT: -pdb:linkrepro-pdb.test.tmp.pdb
+RSP-NEXT: -nodefaultlib
+RSP-NOT: -natvis:/
+RSP-NOT: -pdbstream:srcsrv=/
 
 RUN: yaml2obj %p/Inputs/export.yaml -o %t1.obj
-RUN: lld-link /out:%t1.dll /dll %t1.obj /implib:%t1.lib /export:exportfn1 /export:exportfn2 /linkrepro:.
-RUN: tar xf repro.tar
-RUN: cat repro/response.txt | FileCheck -check-prefix=IMP %s
+RUN: lld-link /out:%t1.dll /dll %t1.obj /implib:%t1.lib /export:exportfn1 /export:exportfn2 /reproduce:repro2.tar
+RUN: tar xf repro2.tar
+RUN: cat repro2/response.txt | FileCheck -check-prefix=IMP %s
 
 IMP: /implib:linkrepro-pdb.test.tmp1.lib

lld/test/COFF/linkrepro.test

@@ -1,9 +1,13 @@
 # REQUIRES: x86, shell
 
 # RUN: rm -rf %t.dir
-# RUN: mkdir -p %t.dir/build1 %t.dir/build2 %t.dir/build3 %t.dir/build4
 # RUN: yaml2obj %p/Inputs/hello32.yaml -o %t.obj
+# RUN: echo '_main@0' > %t.order
+# RUN: touch %t.def
+# RUN: touch %t.cg
 
+Test link.exe-style /linkrepro: flag.
+# RUN: mkdir -p %t.dir/build1
 # RUN: cd %t.dir/build1
 # RUN: lld-link %t.obj %p/Inputs/std32.lib /subsystem:console \
 # RUN:   /entry:main@0 /linkrepro:. /out:%t.exe
@@ -12,7 +16,7 @@
 # RUN: diff %p/Inputs/std32.lib repro/%:p/Inputs/std32.lib
 # RUN: FileCheck %s --check-prefix=RSP < repro/response.txt
 
-# RUN: cd %t.dir/build1
+Test lld-style /reproduce: flag.
 # RUN: lld-link %t.obj %p/Inputs/std32.lib /subsystem:console \
 # RUN:   /entry:main@0 /reproduce:repro2.tar /out:%t.exe
 # RUN: tar xf repro2.tar
@@ -20,6 +24,8 @@
 # RUN: diff %p/Inputs/std32.lib repro2/%:p/Inputs/std32.lib
 # RUN: FileCheck %s --check-prefix=RSP < repro2/response.txt
 
+Test LLD_REPRODUCE env var.
+# RUN: mkdir -p %t.dir/build2
 # RUN: cd %t.dir/build2
 # RUN: env LLD_REPRODUCE=repro.tar lld-link %t.obj %p/Inputs/std32.lib \
 # RUN:    /subsystem:console /entry:main@0 /out:%t.exe
@@ -28,26 +34,65 @@
 # RUN: diff %p/Inputs/std32.lib repro/%:p/Inputs/std32.lib
 # RUN: FileCheck %s --check-prefix=RSP < repro/response.txt
 
+Test adding .lib files with /libpath: to repro archive,
+and various other flags.
+# RUN: mkdir -p %t.dir/build3
 # RUN: cd %t.dir/build3
 # RUN: lld-link %t.obj /libpath:%p/Inputs /defaultlib:std32 /subsystem:console \
-# RUN:   /entry:main@0 /linkrepro:. /out:%t.exe
+# RUN:   /entry:main@0 /linkrepro:. /out:%t.exe /order:@%t.order /def:%t.def
+# RUN: tar tf repro.tar | FileCheck --check-prefix=LIST %s
 # RUN: tar xf repro.tar
 # RUN: diff %t.obj repro/%:t.obj
+# RUN: diff %t.order repro/%:t.order
+# RUN: diff %t.def repro/%:t.def
 # RUN: diff %p/Inputs/std32.lib repro/%:p/Inputs/std32.lib
 # RUN: FileCheck %s --check-prefix=RSP < repro/response.txt
+# RUN: cd repro; lld-link @response.txt
 
+Test adding .lib files with LIB env var to repro archive,
+and various other flags.
+# RUN: mkdir -p %t.dir/build4
 # RUN: cd %t.dir/build4
 # RUN: env LIB=%p/Inputs lld-link %t.obj /defaultlib:std32 /subsystem:console \
-# RUN:   /entry:main@0 /linkrepro:. /out:%t.exe
+# RUN:   /entry:main@0 /linkrepro:. /out:%t.exe /order:@%t.order /def:%t.def
+# RUN: tar tf repro.tar | FileCheck --check-prefix=LIST %s
 # RUN: tar xf repro.tar
 # RUN: diff %t.obj repro/%:t.obj
+# RUN: diff %t.order repro/%:t.order
+# RUN: diff %t.def repro/%:t.def
 # RUN: diff %p/Inputs/std32.lib repro/%:p/Inputs/std32.lib
 # RUN: FileCheck %s --check-prefix=RSP < repro/response.txt
+# RUN: cd repro; lld-link @response.txt
+
+# LIST: .obj
+# LIST: std32.lib
+# LIST: response.txt
+# LIST: .def
+# LIST: .order
 
 # RSP: /subsystem:console
 # RSP: /entry:main@0
-# RSP-NOT: /linkrepro:
 # RSP: /out:
+# RSP-NOT: /order:@/
+# RSP-NOT: /def:/
 # RSP: linkrepro.test.tmp.obj
 # RSP-NOT: defaultlib
 # RSP: std32.lib
+
+Test /call-graph-ordering-file (can't be used with /order:, needs separate test)
+# RUN: mkdir -p %t.dir/build5
+# RUN: cd %t.dir/build5
+# RUN: lld-link %t.obj %p/Inputs/std32.lib /subsystem:console \
+# RUN:     /entry:main@0 /linkrepro:. /out:%t.exe /call-graph-ordering-file:%t.cg
+# RUN: tar tf repro.tar | FileCheck --check-prefix=LISTCG %s
+# RUN: tar xf repro.tar
+# RUN: diff %t.obj repro/%:t.obj
+# RUN: diff %t.cg repro/%:t.cg
+# RUN: FileCheck %s --check-prefix=RSPCG < repro/response.txt
+# RUN: cd repro; lld-link @response.txt
+
+# LISTCG: .obj
+# LISTCG: response.txt
+# LISTCG: .cg
+
+# RSPCG-NOT: /call-graph-ordering-file:/