[XRay] Add the llvm-xray fdr-dump implementation

Summary:
In this change, we implement a `BlockPrinter` which orders records in a
Block that's been indexed by the `BlockIndexer`. This is used in the
`llvm-xray fdr-dump` tool which ties together the various types and
utilities we've been working on, to allow for inspection of XRay FDR
mode traces both with and without verification.

This change is the final step of the refactoring of D50441.

Reviewers: mboerger, eizan

Subscribers: mgorny, hiraditya, llvm-commits

Differential Revision: https://reviews.llvm.org/D51846

llvm-svn: 341887

include/llvm/XRay/BlockPrinter.h

@@ -0,0 +1,60 @@
+//===- BlockPrinter.h - FDR Block Pretty Printer -------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// An implementation of the RecordVisitor which formats a block of records for
+// easier human consumption.
+//
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_INCLUDE_LLVM_XRAY_BLOCKPRINTER_H_
+#define LLVM_INCLUDE_LLVM_XRAY_BLOCKPRINTER_H_
+
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/XRay/FDRRecords.h"
+#include "llvm/XRay/RecordPrinter.h"
+
+namespace llvm {
+namespace xray {
+
+class BlockPrinter : public RecordVisitor {
+  enum class State {
+    Start,
+    Preamble,
+    Metadata,
+    Function,
+    Arg,
+    CustomEvent,
+    End,
+  };
+
+  raw_ostream &OS;
+  RecordPrinter &RP;
+  State CurrentState = State::Start;
+
+public:
+  explicit BlockPrinter(raw_ostream &O, RecordPrinter &P)
+      : RecordVisitor(), OS(O), RP(P) {}
+
+  Error visit(BufferExtents &) override;
+  Error visit(WallclockRecord &) override;
+  Error visit(NewCPUIDRecord &) override;
+  Error visit(TSCWrapRecord &) override;
+  Error visit(CustomEventRecord &) override;
+  Error visit(CallArgRecord &) override;
+  Error visit(PIDRecord &) override;
+  Error visit(NewBufferRecord &) override;
+  Error visit(EndBufferRecord &) override;
+  Error visit(FunctionRecord &) override;
+
+  void reset() { CurrentState = State::Start; }
+};
+
+} // namespace xray
+} // namespace llvm
+
+#endif // LLVM_INCLUDE_LLVM_XRAY_BLOCKPRINTER_H_

lib/XRay/BlockPrinter.cpp

@@ -0,0 +1,96 @@
+//===- BlockPrinter.cpp - FDR Block Pretty Printer Implementation --------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+#include "llvm/XRay/BlockPrinter.h"
+
+namespace llvm {
+namespace xray {
+
+Error BlockPrinter::visit(BufferExtents &R) {
+  OS << "\n[New Block]\n";
+  CurrentState = State::Preamble;
+  return RP.visit(R);
+}
+
+// Preamble printing.
+Error BlockPrinter::visit(NewBufferRecord &R) {
+  if (CurrentState == State::Start)
+    OS << "\n[New Block]\n";
+
+  OS << "Preamble: \n";
+  CurrentState = State::Preamble;
+  return RP.visit(R);
+}
+
+Error BlockPrinter::visit(WallclockRecord &R) {
+  CurrentState = State::Preamble;
+  return RP.visit(R);
+}
+
+Error BlockPrinter::visit(PIDRecord &R) {
+  CurrentState = State::Preamble;
+  return RP.visit(R);
+}
+
+// Metadata printing.
+Error BlockPrinter::visit(NewCPUIDRecord &R) {
+  if (CurrentState == State::Preamble)
+    OS << "\nBody:\n";
+  if (CurrentState == State::Function)
+    OS << "\nMetadata: ";
+  CurrentState = State::Metadata;
+  OS << " ";
+  auto E = RP.visit(R);
+  return E;
+}
+
+Error BlockPrinter::visit(TSCWrapRecord &R) {
+  if (CurrentState == State::Function)
+    OS << "\nMetadata:";
+  CurrentState = State::Metadata;
+  OS << " ";
+  auto E = RP.visit(R);
+  return E;
+}
+
+// Custom events will be rendered like "function" events.
+Error BlockPrinter::visit(CustomEventRecord &R) {
+  if (CurrentState == State::Metadata)
+    OS << "\n";
+  CurrentState = State::CustomEvent;
+  OS << "*  ";
+  auto E = RP.visit(R);
+  return E;
+}
+
+// Function call printing.
+Error BlockPrinter::visit(FunctionRecord &R) {
+  if (CurrentState == State::Metadata)
+    OS << "\n";
+  CurrentState = State::Function;
+  OS << "-  ";
+  auto E = RP.visit(R);
+  return E;
+}
+
+Error BlockPrinter::visit(CallArgRecord &R) {
+  CurrentState = State::Arg;
+  OS << " : ";
+  auto E = RP.visit(R);
+  return E;
+}
+
+Error BlockPrinter::visit(EndBufferRecord &R) {
+    CurrentState = State::End;
+    OS << " *** ";
+    auto E = RP.visit(R);
+    return E;
+}
+
+} // namespace xray
+} // namespace llvm

lib/XRay/CMakeLists.txt

@@ -1,5 +1,6 @@
 add_llvm_library(LLVMXRay
   BlockIndexer.cpp
+  BlockPrinter.cpp
   BlockVerifier.cpp
   FDRRecordProducer.cpp
   FDRRecords.cpp

test/tools/llvm-xray/X86/fdr-dump-arg1-version-3.txt

@@ -0,0 +1,25 @@
+; RUN: llvm-xray fdr-dump -verify %S/Inputs/fdr-log-arg1-version-3.xray \
+; RUN:     | FileCheck %s
+
+; CHECK:      [New Block]
+; CHECK-NEXT: Preamble:
+; CHECK-NEXT: <Thread ID: 2631>
+; CHECK-NEXT: <Wall Time: seconds = 599605.032403>
+; CHECK-NEXT: <PID: 2631>
+; CHECK-EMPTY:
+; CHECK-NEXT: Body:
+; CHECK-NEXT:  <CPU ID: 6>
+; CHECK-NEXT:  <CPU ID: 6>
+; CHECK-NEXT:  <TSC Wrap: base = 2034042117104344>
+; CHECK-EMPTY:
+; CHECK-NEXT: -  <Function Enter: #3 delta = +3>
+; CHECK-NEXT: -  <Function Exit: #3 delta = +3>
+; CHECK-NEXT: -  <Function Enter: #2 delta = +2>
+; CHECK-NEXT: -  <Function Exit: #2 delta = +2>
+; CHECK-EMPTY:
+; CHECK-NEXT: Metadata: <TSC Wrap: base = 2034049739853430>
+; CHECK-EMPTY:
+; CHECK-NEXT: -  <Function Enter: #1 delta = +1>
+; CHECK-NEXT:  : <Call Argument: data = 67 (hex = 0x43)>
+; CHECK-NEXT: -  <Function Exit: #1 delta = +1>
+

test/tools/llvm-xray/X86/fdr-dump-arg1.txt

@@ -0,0 +1,16 @@
+; RUN: llvm-xray fdr-dump -verify %S/Inputs/fdr-log-arg1.xray | FileCheck %s
+
+; CHECK:      [New Block]
+; CHECK-NEXT: Preamble:
+; CHECK-NEXT: <Thread ID: 14648>
+; CHECK-NEXT: <Wall Time: seconds = 1452786.250689>
+; CHECK-EMPTY:
+; CHECK-NEXT: Body:
+; CHECK-NEXT:  <CPU ID: 49>
+; CHECK-NEXT:  <TSC Wrap: base = 18828908666540172>
+; CHECK-EMPTY:
+; CHECK-NEXT: -  <Function Enter: #1 delta = +1>
+; CHECK-NEXT:  : <Call Argument: data = 1 (hex = 0x1)>
+; CHECK-NEXT: -  <Function Exit: #1 delta = +1>
+; CHECK-NEXT:  *** <End of Buffer>
+

tools/llvm-xray/CMakeLists.txt

@@ -14,6 +14,7 @@ add_llvm_tool(llvm-xray
   xray-color-helper.cpp
   xray-converter.cpp
   xray-extract.cpp
+  xray-fdr-dump.cpp
   xray-graph-diff.cpp
   xray-graph.cpp
   xray-registry.cpp

tools/llvm-xray/xray-fdr-dump.cpp

@@ -0,0 +1,119 @@
+//===- xray-fdr-dump.cpp: XRay FDR Trace Dump Tool ------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Implements the FDR trace dumping tool, using the libraries for handling FDR
+// mode traces specifically.
+//
+//===----------------------------------------------------------------------===//
+#include "xray-registry.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/XRay/BlockIndexer.h"
+#include "llvm/XRay/BlockPrinter.h"
+#include "llvm/XRay/BlockVerifier.h"
+#include "llvm/XRay/FDRRecordConsumer.h"
+#include "llvm/XRay/FDRRecordProducer.h"
+#include "llvm/XRay/FDRRecords.h"
+#include "llvm/XRay/FileHeaderReader.h"
+#include "llvm/XRay/RecordPrinter.h"
+
+using namespace llvm;
+using namespace xray;
+
+static cl::SubCommand Dump("fdr-dump", "FDR Trace Dump");
+static cl::opt<std::string> DumpInput(cl::Positional,
+                                      cl::desc("<xray fdr mode log>"),
+                                      cl::Required, cl::sub(Dump));
+static cl::opt<bool> DumpVerify("verify",
+                                cl::desc("verify structure of the log"),
+                                cl::init(false), cl::sub(Dump));
+
+static CommandRegistration Unused(&Dump, []() -> Error {
+  // Open the file provided.
+  int Fd;
+  if (auto EC = sys::fs::openFileForRead(DumpInput, Fd))
+    return createStringError(EC, "Cannot open file '%s' for read.",
+                             DumpInput.c_str());
+
+  uint64_t FileSize;
+  if (auto EC = sys::fs::file_size(DumpInput, FileSize))
+    return createStringError(EC, "Failed to get file size for '%s'.",
+                             DumpInput.c_str());
+
+  std::error_code EC;
+  sys::fs::mapped_file_region MappedFile(
+      Fd, sys::fs::mapped_file_region::mapmode::readonly, FileSize, 0, EC);
+
+  DataExtractor DE(StringRef(MappedFile.data(), MappedFile.size()), true, 8);
+  uint32_t OffsetPtr = 0;
+
+  auto FileHeaderOrError = readBinaryFormatHeader(DE, OffsetPtr);
+  if (!FileHeaderOrError)
+    return FileHeaderOrError.takeError();
+  auto &H = FileHeaderOrError.get();
+
+  FileBasedRecordProducer P(H, DE, OffsetPtr);
+
+  RecordPrinter RP(outs(), "\n");
+  if (!DumpVerify) {
+    PipelineConsumer C({&RP});
+    while (DE.isValidOffsetForDataOfSize(OffsetPtr, 1)) {
+      auto R = P.produce();
+      if (!R)
+        return R.takeError();
+      if (auto E = C.consume(std::move(R.get())))
+        return E;
+    }
+    return Error::success();
+  }
+
+  BlockPrinter BP(outs(), RP);
+  std::vector<std::unique_ptr<Record>> Records;
+  LogBuilderConsumer C(Records);
+  while (DE.isValidOffsetForDataOfSize(OffsetPtr, 1)) {
+    auto R = P.produce();
+    if (!R) {
+      // Print records we've found so far.
+      for (auto &Ptr : Records)
+        if (auto E = Ptr->apply(RP))
+          return joinErrors(std::move(E), R.takeError());
+      return R.takeError();
+    }
+    if (auto E = C.consume(std::move(R.get())))
+      return E;
+  }
+
+  // Once we have a trace, we then index the blocks.
+  BlockIndexer::Index Index;
+  BlockIndexer BI(Index);
+  for (auto &Ptr : Records)
+    if (auto E = Ptr->apply(BI))
+      return E;
+
+  if (auto E = BI.flush())
+    return E;
+
+  // Then we validate while printing each block.
+  BlockVerifier BV;
+  for (auto ProcessThreadBlocks : Index) {
+    auto &Blocks = ProcessThreadBlocks.second;
+    for (auto &B : Blocks) {
+      for (auto *R : B.Records) {
+        if (auto E = R->apply(BV))
+          return E;
+        if (auto E = R->apply(BP))
+          return E;
+      }
+      BV.reset();
+      BP.reset();
+    }
+  }
+  outs().flush();
+  return Error::success();
+});

unittests/XRay/CMakeLists.txt

@@ -5,13 +5,14 @@ set(LLVM_LINK_COMPONENTS
   )
 
 add_llvm_unittest(XRayTests
-  ProfileTest.cpp
   FDRBlockIndexerTest.cpp
   FDRBlockVerifierTest.cpp
   FDRProducerConsumerTest.cpp
   FDRRecordPrinterTest.cpp
+  FDRRecordsTest.cpp
   FDRTraceWriterTest.cpp
   GraphTest.cpp
+  ProfileTest.cpp
   )
 
 add_dependencies(XRayTests intrinsics_gen)

unittests/XRay/FDRRecordsTest.cpp

@@ -0,0 +1,156 @@
+//===- FDRRecords.cpp - Unit Tests for XRay FDR Record Loading ------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+
+#include "llvm/XRay/BlockIndexer.h"
+#include "llvm/XRay/BlockPrinter.h"
+#include "llvm/XRay/BlockVerifier.h"
+#include "llvm/XRay/FDRLogBuilder.h"
+#include "llvm/XRay/FDRRecords.h"
+#include "llvm/XRay/RecordPrinter.h"
+
+namespace llvm {
+namespace xray {
+namespace {
+
+using ::testing::Eq;
+using ::testing::Not;
+
+TEST(XRayFDRTest, BuilderAndBlockIndexer) {
+  // We recreate a single block of valid records, then ensure that we find all
+  // of them belonging in the same index. We do this for three blocks, and
+  // ensure we find the same records in the blocks we deduce.
+  auto Block0 = LogBuilder()
+                    .add<BufferExtents>(100)
+                    .add<NewBufferRecord>(1)
+                    .add<WallclockRecord>(1, 1)
+                    .add<PIDRecord>(1)
+                    .add<FunctionRecord>(RecordTypes::ENTER, 1, 1)
+                    .add<FunctionRecord>(RecordTypes::EXIT, 1, 100)
+                    .consume();
+  auto Block1 = LogBuilder()
+                    .add<BufferExtents>(100)
+                    .add<NewBufferRecord>(1)
+                    .add<WallclockRecord>(1, 2)
+                    .add<PIDRecord>(1)
+                    .add<FunctionRecord>(RecordTypes::ENTER, 1, 1)
+                    .add<FunctionRecord>(RecordTypes::EXIT, 1, 100)
+                    .consume();
+  auto Block2 = LogBuilder()
+                    .add<BufferExtents>(100)
+                    .add<NewBufferRecord>(2)
+                    .add<WallclockRecord>(1, 3)
+                    .add<PIDRecord>(1)
+                    .add<FunctionRecord>(RecordTypes::ENTER, 1, 1)
+                    .add<FunctionRecord>(RecordTypes::EXIT, 1, 100)
+                    .consume();
+  BlockIndexer::Index Index;
+  BlockIndexer Indexer(Index);
+  for (auto B : {std::ref(Block0), std::ref(Block1), std::ref(Block2)}) {
+    for (auto &R : B.get())
+      ASSERT_FALSE(errorToBool(R->apply(Indexer)));
+    ASSERT_FALSE(errorToBool(Indexer.flush()));
+  }
+
+  // We have two threads worth of blocks.
+  ASSERT_THAT(Index.size(), Eq(2u));
+  auto T1Blocks = Index.find({1, 1});
+  ASSERT_THAT(T1Blocks, Not(Eq(Index.end())));
+  ASSERT_THAT(T1Blocks->second.size(), Eq(2u));
+  auto T2Blocks = Index.find({1, 2});
+  ASSERT_THAT(T2Blocks, Not(Eq(Index.end())));
+  ASSERT_THAT(T2Blocks->second.size(), Eq(1u));
+}
+
+TEST(XRayFDRTest, BuilderAndBlockVerifier) {
+  auto Block = LogBuilder()
+                   .add<BufferExtents>(48)
+                   .add<NewBufferRecord>(1)
+                   .add<WallclockRecord>(1, 1)
+                   .add<PIDRecord>(1)
+                   .add<NewCPUIDRecord>(1)
+                   .consume();
+  BlockVerifier Verifier;
+  for (auto &R : Block)
+    ASSERT_FALSE(errorToBool(R->apply(Verifier)));
+  ASSERT_FALSE(errorToBool(Verifier.verify()));
+}
+
+TEST(XRayFDRTest, IndexAndVerifyBlocks) {
+  auto Block0 = LogBuilder()
+                    .add<BufferExtents>(64)
+                    .add<NewBufferRecord>(1)
+                    .add<WallclockRecord>(1, 1)
+                    .add<PIDRecord>(1)
+                    .add<NewCPUIDRecord>(1)
+                    .add<FunctionRecord>(RecordTypes::ENTER, 1, 1)
+                    .add<FunctionRecord>(RecordTypes::EXIT, 1, 100)
+                    .consume();
+  auto Block1 = LogBuilder()
+                    .add<BufferExtents>(64)
+                    .add<NewBufferRecord>(1)
+                    .add<WallclockRecord>(1, 1)
+                    .add<PIDRecord>(1)
+                    .add<NewCPUIDRecord>(1)
+                    .add<FunctionRecord>(RecordTypes::ENTER, 1, 1)
+                    .add<FunctionRecord>(RecordTypes::EXIT, 1, 100)
+                    .consume();
+  auto Block2 = LogBuilder()
+                    .add<BufferExtents>(64)
+                    .add<NewBufferRecord>(1)
+                    .add<WallclockRecord>(1, 1)
+                    .add<PIDRecord>(1)
+                    .add<NewCPUIDRecord>(1)
+                    .add<FunctionRecord>(RecordTypes::ENTER, 1, 1)
+                    .add<FunctionRecord>(RecordTypes::EXIT, 1, 100)
+                    .consume();
+
+  // First, index the records in different blocks.
+  BlockIndexer::Index Index;
+  BlockIndexer Indexer(Index);
+  for (auto B : {std::ref(Block0), std::ref(Block1), std::ref(Block2)}) {
+    for (auto &R : B.get())
+      ASSERT_FALSE(errorToBool(R->apply(Indexer)));
+    ASSERT_FALSE(errorToBool(Indexer.flush()));
+  }
+
+  // Next, verify that each block is consistently defined.
+  BlockVerifier Verifier;
+  for (auto &ProcessThreadBlocks : Index) {
+    auto &Blocks = ProcessThreadBlocks.second;
+    for (auto &B : Blocks) {
+      for (auto *R : B.Records)
+        ASSERT_FALSE(errorToBool(R->apply(Verifier)));
+      ASSERT_FALSE(errorToBool(Verifier.verify()));
+      Verifier.reset();
+    }
+  }
+
+  // Then set up the printing mechanisms.
+  std::string Output;
+  raw_string_ostream OS(Output);
+  RecordPrinter RP(OS);
+  BlockPrinter BP(OS, RP);
+  for (auto &ProcessThreadBlocks : Index) {
+    auto &Blocks = ProcessThreadBlocks.second;
+    for (auto &B : Blocks) {
+      for (auto *R : B.Records)
+        ASSERT_FALSE(errorToBool(R->apply(BP)));
+      BP.reset();
+    }
+  }
+
+  OS.flush();
+  EXPECT_THAT(Output, Not(Eq("")));
+}
+
+} // namespace
+} // namespace xray
+} // namespace llvm