[lldb/Reproducers] Support multiple GDB remotes

When running the test suite with always capture on, a handful of tests
are failing because they have multiple targets and therefore multiple
GDB remote connections. The current reproducer infrastructure is capable
of dealing with that.

This patch reworks the GDB remote provider to support multiple GDB
remote connections, similar to how the reproducers support shadowing
multiple command interpreter inputs. The provider now keeps a list of
packet recorders which deal with a single GDB remote connection. During
replay we rely on the order of creation to match the number of packets
to the GDB remote connection.

Differential revision: https://reviews.llvm.org/D71105

lldb/include/lldb/Utility/GDBRemote.h

@@ -9,6 +9,8 @@
 #ifndef liblldb_GDBRemote_h_
 #define liblldb_GDBRemote_h_
 
+#include "lldb/Utility/FileSpec.h"
+#include "lldb/Utility/Reproducer.h"
 #include "lldb/Utility/StreamString.h"
 #include "lldb/lldb-enumerations.h"
 #include "lldb/lldb-public.h"
@@ -69,7 +71,6 @@ struct GDBRemotePacket {
     std::string data;
   };
 
-  void Serialize(llvm::raw_ostream &strm) const;
   void Dump(Stream &strm) const;
 
   BinaryData packet;
@@ -82,6 +83,46 @@ private:
   llvm::StringRef GetTypeStr() const;
 };
 
+namespace repro {
+class PacketRecorder : public AbstractRecorder {
+public:
+  PacketRecorder(const FileSpec &filename, std::error_code &ec)
+      : AbstractRecorder(filename, ec) {}
+
+  static llvm::Expected<std::unique_ptr<PacketRecorder>>
+  Create(const FileSpec &filename);
+
+  void Record(const GDBRemotePacket &packet);
+};
+
+class GDBRemoteProvider : public repro::Provider<GDBRemoteProvider> {
+public:
+  struct Info {
+    static const char *name;
+    static const char *file;
+  };
+
+  GDBRemoteProvider(const FileSpec &directory) : Provider(directory) {}
+
+  llvm::raw_ostream *GetHistoryStream();
+  PacketRecorder *GetNewPacketRecorder();
+
+  void SetCallback(std::function<void()> callback) {
+    m_callback = std::move(callback);
+  }
+
+  void Keep() override;
+  void Discard() override;
+
+  static char ID;
+
+private:
+  std::function<void()> m_callback;
+  std::unique_ptr<llvm::raw_fd_ostream> m_stream_up;
+  std::vector<std::unique_ptr<PacketRecorder>> m_packet_recorders;
+};
+
+} // namespace repro
 } // namespace lldb_private
 
 LLVM_YAML_IS_DOCUMENT_LIST_VECTOR(lldb_private::GDBRemotePacket)

lldb/include/lldb/Utility/Reproducer.h

@@ -153,12 +153,33 @@ public:
   static char ID;
 };
 
-class DataRecorder {
-public:
-  DataRecorder(const FileSpec &filename, std::error_code &ec)
+class AbstractRecorder {
+protected:
+  AbstractRecorder(const FileSpec &filename, std::error_code &ec)
       : m_filename(filename.GetFilename().GetStringRef()),
         m_os(filename.GetPath(), ec, llvm::sys::fs::OF_Text), m_record(true) {}
 
+public:
+  const FileSpec &GetFilename() { return m_filename; }
+
+  void Stop() {
+    assert(m_record);
+    m_record = false;
+  }
+
+private:
+  FileSpec m_filename;
+
+protected:
+  llvm::raw_fd_ostream m_os;
+  bool m_record;
+};
+
+class DataRecorder : public AbstractRecorder {
+public:
+  DataRecorder(const FileSpec &filename, std::error_code &ec)
+      : AbstractRecorder(filename, ec) {}
+
   static llvm::Expected<std::unique_ptr<DataRecorder>>
   Create(const FileSpec &filename);
 
@@ -170,18 +191,6 @@ public:
       m_os << '\n';
     m_os.flush();
   }
-
-  const FileSpec &GetFilename() { return m_filename; }
-
-  void Stop() {
-    assert(m_record);
-    m_record = false;
-  }
-
-private:
-  FileSpec m_filename;
-  llvm::raw_fd_ostream m_os;
-  bool m_record;
 };
 
 class CommandProvider : public Provider<CommandProvider> {
@@ -204,32 +213,6 @@ private:
   std::vector<std::unique_ptr<DataRecorder>> m_data_recorders;
 };
 
-class ProcessGDBRemoteProvider
-    : public repro::Provider<ProcessGDBRemoteProvider> {
-public:
-  struct Info {
-    static const char *name;
-    static const char *file;
-  };
-
-  ProcessGDBRemoteProvider(const FileSpec &directory) : Provider(directory) {}
-
-  llvm::raw_ostream *GetHistoryStream();
-
-  void SetCallback(std::function<void()> callback) {
-    m_callback = std::move(callback);
-  }
-
-  void Keep() override { m_callback(); }
-  void Discard() override { m_callback(); }
-
-  static char ID;
-
-private:
-  std::function<void()> m_callback;
-  std::unique_ptr<llvm::raw_fd_ostream> m_stream_up;
-};
-
 /// The generator is responsible for the logic needed to generate a
 /// reproducer. For doing so it relies on providers, who serialize data that
 /// is necessary for reproducing  a failure.
@@ -359,13 +342,43 @@ private:
   mutable std::mutex m_mutex;
 };
 
-/// Helper class for replaying commands through the reproducer.
-class CommandLoader {
+template <typename T> class MultiLoader {
 public:
-  CommandLoader(std::vector<std::string> files) : m_files(files) {}
+  MultiLoader(std::vector<std::string> files) : m_files(files) {}
 
-  static std::unique_ptr<CommandLoader> Create(Loader *loader);
-  llvm::Optional<std::string> GetNextFile();
+  static std::unique_ptr<MultiLoader> Create(Loader *loader) {
+    if (!loader)
+      return {};
+
+    FileSpec file = loader->GetFile<typename T::Info>();
+    if (!file)
+      return {};
+
+    auto error_or_file = llvm::MemoryBuffer::getFile(file.GetPath());
+    if (auto err = error_or_file.getError())
+      return {};
+
+    std::vector<std::string> files;
+    llvm::yaml::Input yin((*error_or_file)->getBuffer());
+    yin >> files;
+
+    if (auto err = yin.error())
+      return {};
+
+    for (auto &file : files) {
+      FileSpec absolute_path =
+          loader->GetRoot().CopyByAppendingPathComponent(file);
+      file = absolute_path.GetPath();
+    }
+
+    return std::make_unique<MultiLoader<T>>(std::move(files));
+  }
+
+  llvm::Optional<std::string> GetNextFile() {
+    if (m_index >= m_files.size())
+      return {};
+    return m_files[m_index++];
+  }
 
 private:
   std::vector<std::string> m_files;

lldb/source/API/SBDebugger.cpp

@@ -315,8 +315,9 @@ SBError SBDebugger::SetInputFile(SBFile file) {
 
   FileSP file_sp = file.m_opaque_sp;
 
-  static std::unique_ptr<repro::CommandLoader> loader =
-      repro::CommandLoader::Create(repro::Reproducer::Instance().GetLoader());
+  static std::unique_ptr<repro::MultiLoader<repro::CommandProvider>> loader =
+      repro::MultiLoader<repro::CommandProvider>::Create(
+          repro::Reproducer::Instance().GetLoader());
   if (loader) {
     llvm::Optional<std::string> nextfile = loader->GetNextFile();
     FILE *fh = nextfile ? FileSystem::Instance().Fopen(nextfile->c_str(), "r")

lldb/source/Commands/CommandObjectReproducer.cpp

@@ -407,10 +407,9 @@ protected:
       return true;
     }
     case eReproducerProviderCommands: {
-      // Create a new command loader.
-      std::unique_ptr<repro::CommandLoader> command_loader =
-          repro::CommandLoader::Create(loader);
-      if (!command_loader) {
+      std::unique_ptr<repro::MultiLoader<repro::CommandProvider>> multi_loader =
+          repro::MultiLoader<repro::CommandProvider>::Create(loader);
+      if (!multi_loader) {
         SetError(result,
                  make_error<StringError>(llvm::inconvertibleErrorCode(),
                                          "Unable to create command loader."));
@@ -418,9 +417,8 @@ protected:
       }
 
       // Iterate over the command files and dump them.
-      while (true) {
-        llvm::Optional<std::string> command_file =
-            command_loader->GetNextFile();
+      llvm::Optional<std::string> command_file;
+      while ((command_file = multi_loader->GetNextFile())) {
         if (!command_file)
           break;
 
@@ -436,8 +434,12 @@ protected:
       return true;
     }
     case eReproducerProviderGDB: {
-      FileSpec gdb_file = loader->GetFile<ProcessGDBRemoteProvider::Info>();
-      auto error_or_file = MemoryBuffer::getFile(gdb_file.GetPath());
+      std::unique_ptr<repro::MultiLoader<repro::GDBRemoteProvider>>
+          multi_loader =
+              repro::MultiLoader<repro::GDBRemoteProvider>::Create(loader);
+      llvm::Optional<std::string> gdb_file;
+      while ((gdb_file = multi_loader->GetNextFile())) {
+        auto error_or_file = MemoryBuffer::getFile(*gdb_file);
         if (auto err = error_or_file.getError()) {
           SetError(result, errorCodeToError(err));
           return false;
@@ -455,6 +457,7 @@ protected:
         for (GDBRemotePacket &packet : packets) {
           packet.Dump(result.GetOutputStream());
         }
+      }
 
       result.SetStatus(eReturnStatusSuccessFinishResult);
       return true;

lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunication.cpp

@@ -31,6 +31,7 @@
 #include "lldb/Utility/FileSpec.h"
 #include "lldb/Utility/Log.h"
 #include "lldb/Utility/RegularExpression.h"
+#include "lldb/Utility/Reproducer.h"
 #include "lldb/Utility/StreamString.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/Support/ScopedPrinter.h"
@@ -1243,8 +1244,9 @@ Status GDBRemoteCommunication::StartDebugserverProcess(
 
 void GDBRemoteCommunication::DumpHistory(Stream &strm) { m_history.Dump(strm); }
 
-void GDBRemoteCommunication::SetHistoryStream(llvm::raw_ostream *strm) {
-  m_history.SetStream(strm);
+void GDBRemoteCommunication::SetPacketRecorder(
+    repro::PacketRecorder *recorder) {
+  m_history.SetRecorder(recorder);
 }
 
 llvm::Error

lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunication.h

@@ -27,6 +27,9 @@
 #include "lldb/lldb-public.h"
 
 namespace lldb_private {
+namespace repro {
+class PacketRecorder;
+}
 namespace process_gdb_remote {
 
 enum GDBStoppointType {
@@ -133,7 +136,8 @@ public:
                          // fork/exec to avoid having to connect/accept
 
   void DumpHistory(Stream &strm);
-  void SetHistoryStream(llvm::raw_ostream *strm);
+
+  void SetPacketRecorder(repro::PacketRecorder *recorder);
 
   static llvm::Error ConnectLocally(GDBRemoteCommunication &client,
                                     GDBRemoteCommunication &server);

lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationHistory.cpp

@@ -40,8 +40,8 @@ void GDBRemoteCommunicationHistory::AddPacket(char packet_char,
   m_packets[idx].bytes_transmitted = bytes_transmitted;
   m_packets[idx].packet_idx = m_total_packet_count;
   m_packets[idx].tid = llvm::get_threadid();
-  if (m_stream)
-    m_packets[idx].Serialize(*m_stream);
+  if (m_recorder)
+    m_recorder->Record(m_packets[idx]);
 }
 
 void GDBRemoteCommunicationHistory::AddPacket(const std::string &src,
@@ -58,8 +58,8 @@ void GDBRemoteCommunicationHistory::AddPacket(const std::string &src,
   m_packets[idx].bytes_transmitted = bytes_transmitted;
   m_packets[idx].packet_idx = m_total_packet_count;
   m_packets[idx].tid = llvm::get_threadid();
-  if (m_stream)
-    m_packets[idx].Serialize(*m_stream);
+  if (m_recorder)
+    m_recorder->Record(m_packets[idx]);
 }
 
 void GDBRemoteCommunicationHistory::Dump(Stream &strm) const {

lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationHistory.h

@@ -13,11 +13,15 @@
 #include <vector>
 
 #include "lldb/Utility/GDBRemote.h"
+#include "lldb/Utility/Reproducer.h"
 #include "lldb/lldb-public.h"
 #include "llvm/Support/YAMLTraits.h"
 #include "llvm/Support/raw_ostream.h"
 
 namespace lldb_private {
+namespace repro {
+class PacketRecorder;
+}
 namespace process_gdb_remote {
 
 /// The history keeps a circular buffer of GDB remote packets. The history is
@@ -41,7 +45,7 @@ public:
   void Dump(Log *log) const;
   bool DidDumpToLog() const { return m_dumped_to_log; }
 
-  void SetStream(llvm::raw_ostream *strm) { m_stream = strm; }
+  void SetRecorder(repro::PacketRecorder *recorder) { m_recorder = recorder; }
 
 private:
   uint32_t GetFirstSavedPacketIndex() const {
@@ -73,7 +77,7 @@ private:
   uint32_t m_curr_idx;
   uint32_t m_total_packet_count;
   mutable bool m_dumped_to_log;
-  llvm::raw_ostream *m_stream = nullptr;
+  repro::PacketRecorder *m_recorder = nullptr;
 };
 
 } // namespace process_gdb_remote

lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp

@@ -279,12 +279,9 @@ ProcessGDBRemote::ProcessGDBRemote(lldb::TargetSP target_sp,
                                    "async thread did exit");
 
   if (repro::Generator *g = repro::Reproducer::Instance().GetGenerator()) {
-    repro::ProcessGDBRemoteProvider &provider =
-        g->GetOrCreate<repro::ProcessGDBRemoteProvider>();
-    // Set the history stream to the stream owned by the provider.
-    m_gdb_comm.SetHistoryStream(provider.GetHistoryStream());
-    // Make sure to clear the stream again when we're finished.
-    provider.SetCallback([&]() { m_gdb_comm.SetHistoryStream(nullptr); });
+    repro::GDBRemoteProvider &provider =
+        g->GetOrCreate<repro::GDBRemoteProvider>();
+    m_gdb_comm.SetPacketRecorder(provider.GetNewPacketRecorder());
   }
 
   Log *log(ProcessGDBRemoteLog::GetLogIfAllCategoriesSet(GDBR_LOG_ASYNC));
@@ -3362,17 +3359,20 @@ Status ProcessGDBRemote::ConnectToReplayServer(repro::Loader *loader) {
   if (!loader)
     return Status("No loader provided.");
 
-  // Construct replay history path.
-  FileSpec history_file =
-      loader->GetFile<repro::ProcessGDBRemoteProvider::Info>();
-  if (!history_file)
-    return Status("No provider for gdb-remote.");
+  static std::unique_ptr<repro::MultiLoader<repro::GDBRemoteProvider>>
+      multi_loader = repro::MultiLoader<repro::GDBRemoteProvider>::Create(
+          repro::Reproducer::Instance().GetLoader());
 
-  // Enable replay mode.
-  m_replay_mode = true;
+  if (!multi_loader)
+    return Status("No gdb remote provider found.");
+
+  llvm::Optional<std::string> history_file = multi_loader->GetNextFile();
+  if (!history_file)
+    return Status("No gdb remote packet log found.");
 
   // Load replay history.
-  if (auto error = m_gdb_replay_server.LoadReplayHistory(history_file))
+  if (auto error =
+          m_gdb_replay_server.LoadReplayHistory(FileSpec(*history_file)))
     return Status("Unable to load replay history");
 
   // Make a local connection.
@@ -3380,6 +3380,9 @@ Status ProcessGDBRemote::ConnectToReplayServer(repro::Loader *loader) {
                                                           m_gdb_replay_server))
     return Status("Unable to connect to replay server");
 
+  // Enable replay mode.
+  m_replay_mode = true;
+
   // Start server thread.
   m_gdb_replay_server.StartAsyncThread();
 

lldb/source/Utility/GDBRemote.cpp

@@ -14,6 +14,7 @@
 #include <stdio.h>
 
 using namespace lldb;
+using namespace lldb_private::repro;
 using namespace lldb_private;
 using namespace llvm;
 
@@ -45,12 +46,6 @@ int StreamGDBRemote::PutEscapedBytes(const void *s, size_t src_len) {
   return bytes_written;
 }
 
-void GDBRemotePacket::Serialize(raw_ostream &strm) const {
-  yaml::Output yout(strm);
-  yout << const_cast<GDBRemotePacket &>(*this);
-  strm.flush();
-}
-
 llvm::StringRef GDBRemotePacket::GetTypeStr() const {
   switch (type) {
   case GDBRemotePacket::ePacketTypeSend:
@@ -103,3 +98,66 @@ yaml::MappingTraits<GDBRemotePacket>::validate(IO &io,
 
   return {};
 }
+
+void GDBRemoteProvider::Keep() {
+  std::vector<std::string> files;
+  for (auto &recorder : m_packet_recorders) {
+    files.push_back(recorder->GetFilename().GetPath());
+  }
+
+  FileSpec file = GetRoot().CopyByAppendingPathComponent(Info::file);
+  std::error_code ec;
+  llvm::raw_fd_ostream os(file.GetPath(), ec, llvm::sys::fs::OF_Text);
+  if (ec)
+    return;
+  yaml::Output yout(os);
+  yout << files;
+}
+
+void GDBRemoteProvider::Discard() { m_packet_recorders.clear(); }
+
+llvm::Expected<std::unique_ptr<PacketRecorder>>
+PacketRecorder::Create(const FileSpec &filename) {
+  std::error_code ec;
+  auto recorder = std::make_unique<PacketRecorder>(std::move(filename), ec);
+  if (ec)
+    return llvm::errorCodeToError(ec);
+  return std::move(recorder);
+}
+
+PacketRecorder *GDBRemoteProvider::GetNewPacketRecorder() {
+  std::size_t i = m_packet_recorders.size() + 1;
+  std::string filename = (llvm::Twine(Info::name) + llvm::Twine("-") +
+                          llvm::Twine(i) + llvm::Twine(".yaml"))
+                             .str();
+  auto recorder_or_error =
+      PacketRecorder::Create(GetRoot().CopyByAppendingPathComponent(filename));
+  if (!recorder_or_error) {
+    llvm::consumeError(recorder_or_error.takeError());
+    return nullptr;
+  }
+
+  m_packet_recorders.push_back(std::move(*recorder_or_error));
+  return m_packet_recorders.back().get();
+}
+
+void PacketRecorder::Record(const GDBRemotePacket &packet) {
+  if (!m_record)
+    return;
+  yaml::Output yout(m_os);
+  yout << const_cast<GDBRemotePacket &>(packet);
+  m_os.flush();
+}
+
+llvm::raw_ostream *GDBRemoteProvider::GetHistoryStream() {
+  FileSpec history_file = GetRoot().CopyByAppendingPathComponent(Info::file);
+
+  std::error_code EC;
+  m_stream_up = std::make_unique<raw_fd_ostream>(history_file.GetPath(), EC,
+                                                 sys::fs::OpenFlags::OF_Text);
+  return m_stream_up.get();
+}
+
+char GDBRemoteProvider::ID = 0;
+const char *GDBRemoteProvider::Info::file = "gdb-remote.yaml";
+const char *GDBRemoteProvider::Info::name = "gdb-remote";

lldb/source/Utility/Reproducer.cpp

@@ -255,7 +255,7 @@ DataRecorder::Create(const FileSpec &filename) {
 DataRecorder *CommandProvider::GetNewDataRecorder() {
   std::size_t i = m_data_recorders.size() + 1;
   std::string filename = (llvm::Twine(Info::name) + llvm::Twine("-") +
-                          llvm::Twine(i) + llvm::Twine(".txt"))
+                          llvm::Twine(i) + llvm::Twine(".yaml"))
                              .str();
   auto recorder_or_error =
       DataRecorder::Create(GetRoot().CopyByAppendingPathComponent(filename));
@@ -304,53 +304,9 @@ void WorkingDirectoryProvider::Keep() {
   os << m_cwd << "\n";
 }
 
-llvm::raw_ostream *ProcessGDBRemoteProvider::GetHistoryStream() {
-  FileSpec history_file = GetRoot().CopyByAppendingPathComponent(Info::file);
-
-  std::error_code EC;
-  m_stream_up = std::make_unique<raw_fd_ostream>(history_file.GetPath(), EC,
-                                                 sys::fs::OpenFlags::OF_Text);
-  return m_stream_up.get();
-}
-
-std::unique_ptr<CommandLoader> CommandLoader::Create(Loader *loader) {
-  if (!loader)
-    return {};
-
-  FileSpec file = loader->GetFile<repro::CommandProvider::Info>();
-  if (!file)
-    return {};
-
-  auto error_or_file = llvm::MemoryBuffer::getFile(file.GetPath());
-  if (auto err = error_or_file.getError())
-    return {};
-
-  std::vector<std::string> files;
-  llvm::yaml::Input yin((*error_or_file)->getBuffer());
-  yin >> files;
-
-  if (auto err = yin.error())
-    return {};
-
-  for (auto &file : files) {
-    FileSpec absolute_path =
-        loader->GetRoot().CopyByAppendingPathComponent(file);
-    file = absolute_path.GetPath();
-  }
-
-  return std::make_unique<CommandLoader>(std::move(files));
-}
-
-llvm::Optional<std::string> CommandLoader::GetNextFile() {
-  if (m_index >= m_files.size())
-    return {};
-  return m_files[m_index++];
-}
-
 void ProviderBase::anchor() {}
 char CommandProvider::ID = 0;
 char FileProvider::ID = 0;
-char ProcessGDBRemoteProvider::ID = 0;
 char ProviderBase::ID = 0;
 char VersionProvider::ID = 0;
 char WorkingDirectoryProvider::ID = 0;
@@ -358,8 +314,6 @@ const char *CommandProvider::Info::file = "command-interpreter.yaml";
 const char *CommandProvider::Info::name = "command-interpreter";
 const char *FileProvider::Info::file = "files.yaml";
 const char *FileProvider::Info::name = "files";
-const char *ProcessGDBRemoteProvider::Info::file = "gdb-remote.yaml";
-const char *ProcessGDBRemoteProvider::Info::name = "gdb-remote";
 const char *VersionProvider::Info::file = "version.txt";
 const char *VersionProvider::Info::name = "version";
 const char *WorkingDirectoryProvider::Info::file = "cwd.txt";

lldb/test/Shell/Reproducer/Inputs/MultipleTargetsCapture.in

@@ -0,0 +1,12 @@
+target select 0
+breakpoint set -f simple.c -l 12
+run
+target select 1
+breakpoint set -f simple.c -l 16
+run
+target select 0
+cont
+target select 1
+cont
+reproducer status
+reproducer generate

lldb/test/Shell/Reproducer/TestMultipleTargets.test

@@ -0,0 +1,23 @@
+# UNSUPPORTED: system-windows, system-freebsd
+
+# This tests the replaying with multiple targets.
+
+# RUN: %clang_host %S/Inputs/simple.c -g -o %t.out
+
+# RUN: rm -rf %t.repro
+# RUN: %lldb -x -b --capture --capture-path %t.repro -o 'target create %t.out' -o 'target create %t.out' -s %S/Inputs/MultipleTargetsCapture.in  | FileCheck %s --check-prefix CHECK --check-prefix CAPTURE
+# RUN: env FOO=BAR %lldb --replay %t.repro | FileCheck %s --check-prefix CHECK --check-prefix REPLAY
+
+# CHECK: Process [[TARGET0:[0-9]+]] stopped
+# CHECK: stop reason = breakpoint 1.1
+# CHECK: simple.c:12:5
+# CHECK: Process [[TARGET1:[0-9]+]] stopped
+# CHECK: stop reason = breakpoint 1.1
+# CHECK: simple.c:16:5
+# CHECK: Process [[TARGET0]] resuming
+# CHECK: Process [[TARGET0]] exited
+# CHECK: Process [[TARGET1]] resuming
+# CHECK: Process [[TARGET1]] exited
+
+# CAPTURE: Reproducer is in capture mode.
+# CAPTURE: Reproducer written