Magic-Mirror/llvm-capstone
1
0
Fork 0
mirror of https://github.com/capstone-engine/llvm-capstone.git synced 2025-02-14 14:56:47 +00:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

[BOLT] Encode instrumentation tables in file

Browse Source 
Summary:
Avoid directly allocating string and description tables in
binary's static data region, since they are not needed during runtime
except when writing the profile at exit. Change the runtime library to
open the tables on disk and read only when necessary.

(cherry picked from FBD16626030)
This commit is contained in:
Rafael Auler 2019-08-02 11:20:13 -07:00 committed by Maksim Panchenko
parent 62aa74f836
commit 821480d27f
8 changed files with 257 additions and 86 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

196
bolt/runtime/instr.cpp
View File

@ -10,11 +10,12 @@
//
//===----------------------------------------------------------------------===//
//
// BOLT runtime instrumentation library.
// BOLT runtime instrumentation library for x86 Linux.
//
//===----------------------------------------------------------------------===//
#include <cstdint>
#include <elf.h>
// All extern declarations here need to be defined by BOLT itself.
@ -23,31 +24,38 @@
extern uint64_t __bolt_instr_locations[];
// Number of counters.
extern uint32_t __bolt_instr_num_locs;
// String table with function names.
extern char __bolt_instr_strings[];
// Filename to dump data to.
extern char __bolt_instr_filename[];
// A location is a function name plus offset. Function name needs to be
// retrieved from the string table and is stored as an index to this table.
typedef struct _Location {
struct Location {
uint32_t FunctionName;
uint32_t Offset;
} Location;
};
// An edge description defines an instrumented edge in the program, fully
// identified by where the jump is located and its destination.
typedef struct _EdgeDescription {
struct EdgeDescription {
Location From;
Location To;
} EdgeDescription;
};
extern EdgeDescription __bolt_instr_descriptions[];
// These need to be read from disk. They are generated by BOLT and written to
// an ELF note section in the binary itself.
struct InstrumentationInfo {
EdgeDescription *Descriptions;
char *Strings; // String table with function names used in this binary
int FileDesc; // File descriptor for the file on disk backing this
// information in memory via mmap
uint8_t *MMapPtr; // The mmap ptr
int MMapSize; // The mmap size
};
// Declare some syscall wrappers we use throughout this code to avoid linking
// against system libc.
static uint64_t
myopen(const char *pathname,
__open(const char *pathname,
uint64_t flags,
uint64_t mode) {
uint64_t ret;
@ -60,7 +68,7 @@ myopen(const char *pathname,
return ret;
}
static uint64_t mywrite(uint64_t fd, const void *buf, uint64_t count) {
static uint64_t __write(uint64_t fd, const void *buf, uint64_t count) {
uint64_t ret;
__asm__ __volatile__ (
"movq $1, %%rax\n"
@ -71,7 +79,18 @@ static uint64_t mywrite(uint64_t fd, const void *buf, uint64_t count) {
return ret;
}
static int myclose(uint64_t fd) {
static uint64_t __lseek(uint64_t fd, uint64_t pos, uint64_t whence) {
uint64_t ret;
__asm__ __volatile__ (
"movq $8, %%rax\n"
"syscall\n"
: "=a"(ret)
: "D"(fd), "S"(pos), "d"(whence)
: "cc", "rcx", "r11", "memory");
return ret;
}
static int __close(uint64_t fd) {
uint64_t ret;
__asm__ __volatile__ (
"movq $3, %%rax\n"
@ -82,6 +101,47 @@ static int myclose(uint64_t fd) {
return ret;
}
static void *__mmap(uint64_t addr, uint64_t size, uint64_t prot,
uint64_t flags, uint64_t fd, uint64_t offset) {
void *ret;
register uint64_t r8 asm("r8") = fd;
register uint64_t r9 asm("r9") = offset;
register uint64_t r10 asm("r10") = flags;
__asm__ __volatile__ (
"movq $9, %%rax\n"
"syscall\n"
: "=a"(ret)
: "D"(addr), "S"(size), "d"(prot), "r"(r10), "r"(r8), "r"(r9)
: "cc", "rcx", "r11", "memory");
return ret;
}
static uint64_t __munmap(void *addr, uint64_t size) {
uint64_t ret;
__asm__ __volatile__ (
"movq $11, %%rax\n"
"syscall\n"
: "=a"(ret)
: "D"(addr), "S"(size)
: "cc", "rcx", "r11", "memory");
return ret;
}
static uint64_t __exit(uint64_t code) {
uint64_t ret;
__asm__ __volatile__ (
"movq $231, %%rax\n"
"syscall\n"
: "=a"(ret)
: "D"(code)
: "cc", "rcx", "r11", "memory");
return ret;
}
// Helper functions for writing strings to the .fdata file
// Write number Num using Base to the buffer in OutBuf, returns a pointer to
// the end of the string.
static char *intToStr(char *OutBuf, uint32_t Num, uint32_t Base) {
const char *Chars = "0123456789abcdef";
char Buf[20];
@ -100,22 +160,107 @@ static char *intToStr(char *OutBuf, uint32_t Num, uint32_t Base) {
return OutBuf;
}
static char *serializeLoc(char *OutBuf, uint32_t FuncStrIndex,
uint32_t Offset) {
*OutBuf++ = '1';
*OutBuf++ = ' ';
char *Str = __bolt_instr_strings + FuncStrIndex;
// Copy Str to OutBuf, returns a pointer to the end of the copied string.
static char *strCopy(char *OutBuf, const char *Str) {
while (*Str)
*OutBuf++ = *Str++;
return OutBuf;
}
// Print Msg to STDERR and quits with error code 1.
static void reportError(const char *Msg, uint64_t Size) {
__write(2, Msg, Size);
__exit(1);
}
// Perform a string comparison and returns zero if Str1 matches Str2. Compares
// at most Size characters.
static int compareStr(const char *Str1, const char *Str2, int Size) {
while (*Str1 == *Str2) {
if (*Str1 == '\0' || --Size == 0)
return 0;
++Str1;
++Str2;
}
return 1;
}
// Write as a string in OutBuf an identifier for the program point at function
// whose name is in the string table index FuncStrIndex plus Offset.
static char *serializeLoc(const InstrumentationInfo &Info, char *OutBuf,
const Location Loc) {
// fdata location format: Type Name Offset
// Type 1 - regular symbol
OutBuf = strCopy(OutBuf, "1 ");
const char *Str = Info.Strings + Loc.FunctionName;
while (*Str) {
*OutBuf++ = *Str++;
}
*OutBuf++ = ' ';
OutBuf = intToStr(OutBuf, Offset, 16);
OutBuf = intToStr(OutBuf, Loc.Offset, 16);
*OutBuf++ = ' ';
return OutBuf;
}
// Read and map to memory the descriptions written by BOLT into the executable's
// notes section
static InstrumentationInfo readDescriptions() {
InstrumentationInfo Result;
uint64_t FD = __open("/proc/self/exe",
/*flags=*/0 /*O_RDONLY*/,
/*mode=*/0666);
Result.FileDesc = FD;
// mmap our binary to memory
uint64_t Size = __lseek(FD, 0, 2 /*SEEK_END*/);
uint8_t *BinContents = reinterpret_cast<uint8_t *>(
__mmap(0, Size, 0x1 /* PROT_READ*/, 0x2 /* MAP_PRIVATE*/, FD, 0));
Result.MMapPtr = BinContents;
Result.MMapSize = Size;
Elf64_Ehdr *Hdr = reinterpret_cast<Elf64_Ehdr *>(BinContents);
Elf64_Shdr *Shdr = reinterpret_cast<Elf64_Shdr *>(BinContents + Hdr->e_shoff);
Elf64_Shdr *StringTblHeader = reinterpret_cast<Elf64_Shdr *>(
BinContents + Hdr->e_shoff + Hdr->e_shstrndx * Hdr->e_shentsize);
// Find .bolt.instr.tables with the data we need and set pointers to it
for (int I = 0; I < Hdr->e_shnum; ++I) {
char *SecName = reinterpret_cast<char *>(
BinContents + StringTblHeader->sh_offset + Shdr->sh_name);
if (compareStr(SecName, ".bolt.instr.tables", 64) != 0) {
Shdr = reinterpret_cast<Elf64_Shdr *>(BinContents + Hdr->e_shoff +
(I + 1) * Hdr->e_shentsize);
continue;
}
// Actual contents of the ELF note start after offset 20 decimal:
// Offset 0: Producer name size (4 bytes)
// Offset 4: Contents size (4 bytes)
// Offset 8: Note type (4 bytes)
// Offset 12: Producer name (BOLT\0) (5 bytes + align to 4-byte boundary)
// Offset 20: Contents
Result.Descriptions =
reinterpret_cast<EdgeDescription *>(BinContents + Shdr->sh_offset + 20);
// String table is located after the full EdgeDescriptions table containing
// __bolt_instr_num_locs entries is finished
Result.Strings = reinterpret_cast<char *>(
BinContents + Shdr->sh_offset + 20 +
(__bolt_instr_num_locs * sizeof(EdgeDescription)));
return Result;
}
const char ErrMsg[] =
"BOLT instrumentation runtime error: could not find section "
".bolt.instr.tables\n";
reportError(ErrMsg, sizeof(ErrMsg));
return Result;
}
// This is the entry point called at program exit. BOLT patches the executable's
// FINI entry in the .dynamic section with the address of this function. Our
// goal here is to flush to disk all instrumentation data in memory, using
// BOLT's fdata format.
extern "C" void __bolt_instr_data_dump() {
uint64_t FD = myopen(__bolt_instr_filename,
const InstrumentationInfo Info = readDescriptions();
uint64_t FD = __open(__bolt_instr_filename,
/*flags=*/0x241 /*O_WRONLY|O_TRUNC|O_CREAT*/,
/*mode=*/0666);
@ -126,14 +271,15 @@ extern "C" void __bolt_instr_data_dump() {
if (!HitCount)
continue;
EdgeDescription *Desc = &__bolt_instr_descriptions[I];
Ptr = serializeLoc(Ptr, Desc->From.FunctionName, Desc->From.Offset);
Ptr = serializeLoc(Ptr, Desc->To.FunctionName, Desc->To.Offset);
*Ptr++ = '0';
*Ptr++ = ' ';
EdgeDescription *Desc = &Info.Descriptions[I];
Ptr = serializeLoc(Info, Ptr, Desc->From);
Ptr = serializeLoc(Info, Ptr, Desc->To);
Ptr = strCopy(Ptr, "0 ");
Ptr = intToStr(Ptr, HitCount, 10);
*Ptr++ = '\n';
mywrite(FD, LineBuf, Ptr - LineBuf);
__write(FD, LineBuf, Ptr - LineBuf);
}
myclose(FD);
__close(FD);
__munmap(Info.MMapPtr, Info.MMapSize);
__close(Info.FileDesc);
}

20
bolt/src/BinarySection.cpp
View File

@ -160,3 +160,23 @@ void BinarySection::reorderContents(const std::vector<BinaryData *> &Order,
Contents = OutputContents = StringRef(NewData, OS.str().size());
OutputSize = Contents.size();
}
std::string BinarySection::encodeELFNote(StringRef NameStr, StringRef DescStr,
uint32_t Type) {
std::string Str;
raw_string_ostream OS(Str);
const uint32_t NameSz = NameStr.size() + 1;
const uint32_t DescSz = DescStr.size();
OS.write(reinterpret_cast<const char *>(&(NameSz)), 4);
OS.write(reinterpret_cast<const char *>(&(DescSz)), 4);
OS.write(reinterpret_cast<const char *>(&(Type)), 4);
OS << NameStr << '\0';
for (uint64_t I = NameSz; I < alignTo(NameSz, 4); ++I) {
OS << '\0';
}
OS << DescStr;
for (uint64_t I = DescStr.size(); I < alignTo(DescStr.size(), 4); ++I) {
OS << '\0';
}
return OS.str();
}

12
bolt/src/BinarySection.h
View File

@ -416,6 +416,18 @@ public:
void reorderContents(const std::vector<BinaryData *> &Order, bool Inplace);
void print(raw_ostream &OS) const;
/// Write the contents of an ELF note section given the name of the producer,
/// a number identifying the type of note and the contents of the note in
/// \p DescStr.
static std::string encodeELFNote(StringRef NameStr, StringRef DescStr,
uint32_t Type);
/// Code for ELF notes written by producer 'BOLT'
enum {
NT_BOLT_BAT = 1,
NT_BOLT_INSTRUMENTATION_TABLES = 2
};
};
inline uint8_t *copyByteArray(const uint8_t *Data, uint64_t Size) {

3
bolt/src/BoltAddressTranslation.cpp
View File

@ -144,8 +144,7 @@ std::error_code BoltAddressTranslation::parse(StringRef Buf) {
const uint32_t DescSz = DE.getU32(&Offset);
const uint32_t Type = DE.getU32(&Offset);
// Note type reserved for BAT
if (Type != 1 ||
if (Type != BinarySection::NT_BOLT_BAT ||
Buf.size() + Offset < alignTo(NameSz, 4) + DescSz)
return make_error_code(llvm::errc::io_error);

2
bolt/src/MCPlusBuilder.cpp
View File

@ -173,7 +173,7 @@ bool MCPlusBuilder::setJumpTable(MCInst &Inst, uint64_t Value,
if (!isIndirectBranch(Inst))
return false;
setAnnotationOpValue(Inst, MCAnnotation::kJumpTable, Value, AllocId);
addAnnotation<>(Inst, "JTIndexReg", IndexReg, AllocId);
getOrCreateAnnotationAs<uint16_t>(Inst, "JTIndexReg", AllocId) = IndexReg;
return true;
}

64
bolt/src/Passes/Instrumentation.cpp
View File

@ -137,10 +137,15 @@ void Instrumentation::runOnFunctions(BinaryContext &BC) {
const auto Flags = BinarySection::getFlags(/*IsReadOnly=*/false,
/*IsText=*/false,
/*IsAllocatable=*/true);
BC.registerOrUpdateSection(".bolt.instrumentation", ELF::SHT_PROGBITS, Flags,
BC.registerOrUpdateSection(".bolt.instr.counters", ELF::SHT_PROGBITS, Flags,
nullptr, 0, 1,
/*local=*/true);
BC.registerOrUpdateNoteSection(".bolt.instr.tables", nullptr,
0,
/*Alignment=*/1,
/*IsReadOnly=*/true, ELF::SHT_NOTE);
uint64_t InstrumentationSites{0ULL};
uint64_t InstrumentationSitesSavingFlags{0ULL};
for (auto &BFI : BC.getBinaryFunctions()) {
@ -237,19 +242,36 @@ void Instrumentation::runOnFunctions(BinaryContext &BC) {
<< " sites, " << InstrumentationSitesSavingFlags << " saving flags.\n";
}
void Instrumentation::emitDescription(
const Instrumentation::CounterDescription &Desc, MCStreamer &Streamer) {
Streamer.EmitIntValue(Desc.FromFuncStringIdx, /*Size=*/4);
Streamer.EmitIntValue(Desc.FromOffset, /*Size=*/4);
Streamer.EmitIntValue(Desc.ToFuncStringIdx, /*Size=*/4);
Streamer.EmitIntValue(Desc.ToOffset, /*Size=*/4);
void Instrumentation::emitTablesAsELFNote(BinaryContext &BC) {
std::string TablesStr;
raw_string_ostream OS(TablesStr);
// Start of the vector with descriptions (one CounterDescription for each
// counter), vector size is Labels.size() CounterDescription-sized elmts
for (const auto &Desc : Descriptions) {
OS.write(reinterpret_cast<const char *>(&Desc.FromFuncStringIdx), 4);
OS.write(reinterpret_cast<const char *>(&Desc.FromOffset), 4);
OS.write(reinterpret_cast<const char *>(&Desc.ToFuncStringIdx), 4);
OS.write(reinterpret_cast<const char *>(&Desc.ToOffset), 4);
}
// Our string table lives immediately after descriptions vector
OS << StringTable;
OS.flush();
const auto BoltInfo = BinarySection::encodeELFNote(
"BOLT", TablesStr, BinarySection::NT_BOLT_INSTRUMENTATION_TABLES);
BC.registerOrUpdateNoteSection(".bolt.instr.tables", copyByteArray(BoltInfo),
BoltInfo.size(),
/*Alignment=*/1,
/*IsReadOnly=*/true, ELF::SHT_NOTE);
}
void Instrumentation::emit(BinaryContext &BC, MCStreamer &Streamer) {
emitTablesAsELFNote(BC);
const auto Flags = BinarySection::getFlags(/*IsReadOnly=*/false,
/*IsText=*/false,
/*IsAllocatable=*/true);
auto *Section = BC.Ctx->getELFSection(".bolt.instrumentation",
auto *Section = BC.Ctx->getELFSection(".bolt.instr.counters",
ELF::SHT_PROGBITS,
Flags);
@ -259,12 +281,6 @@ void Instrumentation::emit(BinaryContext &BC, MCStreamer &Streamer) {
// counters, total vector size is Labels.size() 8-byte counters
MCSymbol *Locs = BC.Ctx->getOrCreateSymbol("__bolt_instr_locations");
MCSymbol *NumLocs = BC.Ctx->getOrCreateSymbol("__bolt_instr_num_locs");
// Start of the vector with descriptions (one CounterDescription for each
// counter), vector size is Labels.size() CounterDescription-sized elmts
MCSymbol *DescriptionsSym =
BC.Ctx->getOrCreateSymbol("__bolt_instr_descriptions");
// Label identifying where our string table was emitted to
MCSymbol *StringsSym = BC.Ctx->getOrCreateSymbol("__bolt_instr_strings");
/// File name where profile is going to written to after target binary
/// finishes a run
MCSymbol *FilenameSym = BC.Ctx->getOrCreateSymbol("__bolt_instr_filename");
@ -281,27 +297,15 @@ void Instrumentation::emit(BinaryContext &BC, MCStreamer &Streamer) {
Streamer.EmitSymbolAttribute(NumLocs,
MCSymbolAttr::MCSA_Global);
Streamer.EmitIntValue(Labels.size(), /*Size=*/4);
Streamer.EmitLabel(DescriptionsSym);
Streamer.EmitSymbolAttribute(DescriptionsSym,
MCSymbolAttr::MCSA_Global);
for (const auto &Desc : Descriptions) {
emitDescription(Desc, Streamer);
}
Streamer.EmitLabel(StringsSym);
Streamer.EmitSymbolAttribute(StringsSym,
MCSymbolAttr::MCSA_Global);
Streamer.EmitBytes(StringTable);
Streamer.EmitLabel(FilenameSym);
Streamer.EmitBytes(opts::InstrumentationFilename);
Streamer.emitFill(1, 0);
outs() << "BOLT-INSTRUMENTER: Total size of string table emitted: "
<< StringTable.size() << " bytes\n";
outs() << "BOLT-INSTRUMENTER: Total size of counters: "
<< (Labels.size() * 8) << " bytes\n";
<< (Labels.size() * 8) << " bytes (static alloc memory)\n";
outs() << "BOLT-INSTRUMENTER: Total size of string table emitted: "
<< StringTable.size() << " bytes in file\n";
outs() << "BOLT-INSTRUMENTER: Total size of descriptors: "
<< (Labels.size() * 16) << " bytes\n";
outs() << "BOLT-INSTRUMENTER: Total data: "
<< (Labels.size() * 24 + StringTable.size()) << " bytes\n";
<< (Labels.size() * 16) << " bytes in file\n";
outs() << "BOLT-INSTRUMENTER: Profile will be saved to file "
<< opts::InstrumentationFilename << "\n";
}

6
bolt/src/Passes/Instrumentation.h
View File

@ -96,7 +96,11 @@ private:
BinaryFunction &ToFunc, BinaryBasicBlock *TargetBB,
uint32_t ToOffset);
void emitDescription(const CounterDescription &Desc, MCStreamer &Streamer);
/// Create a non-allocatable ELF section with read-only tables necessary for
/// writing the instrumented data profile during program finish. The runtime
/// library needs to open the program executable file and read this data from
/// disk, this is not loaded by the system.
void emitTablesAsELFNote(BinaryContext &BC);
/// Critical edges worklist
/// This worklist keeps track of CFG edges <From-To> that needs to be split.

40
bolt/src/RewriteInstance.cpp
View File

@ -3090,7 +3090,14 @@ void RewriteInstance::emitAndLink() {
[&](const std::string &Name) -> JITSymbol {
DEBUG(dbgs() << "BOLT: looking for " << Name << "\n");
if (EFMM->ObjectsLoaded) {
return OLT->findSymbol(Name, false);
auto Result = OLT->findSymbol(Name, false);
if (cantFail(Result.getAddress()) == 0) {
errs()
<< "BOLT-ERROR: symbol not found required by runtime library: "
<< Name << "\n";
exit(1);
}
return Result;
}
if (auto *I = BC->getBinaryDataByName(Name)) {
const uint64_t Address = I->isMoved() && !I->isJumpTable()
@ -3503,7 +3510,7 @@ void RewriteInstance::mapDataSections(orc::VModuleKey Key) {
// The order is important.
std::vector<std::string> Sections = {
".eh_frame", ".eh_frame_old", ".gcc_except_table",
".rodata", ".rodata.cold", ".bolt.instrumentation"};
".rodata", ".rodata.cold", ".bolt.instr.counters"};
for (auto &SectionName : Sections) {
auto Section = BC->getUniqueSectionByName(SectionName);
if (!Section || !Section->isAllocatable() || !Section->isFinalized())
@ -4073,29 +4080,6 @@ void RewriteInstance::finalizeSectionStringTable(ELFObjectFile<ELFT> *File) {
ELF::SHT_STRTAB);
}
namespace {
std::string encodeELFNote(StringRef NameStr, StringRef DescStr, uint32_t Type) {
std::string Str;
raw_string_ostream OS(Str);
const uint32_t NameSz = NameStr.size() + 1;
const uint32_t DescSz = DescStr.size();
OS.write(reinterpret_cast<const char *>(&(NameSz)), 4);
OS.write(reinterpret_cast<const char *>(&(DescSz)), 4);
OS.write(reinterpret_cast<const char *>(&(Type)), 4);
OS << NameStr << '\0';
for (uint64_t I = NameSz; I < alignTo(NameSz, 4); ++I) {
OS << '\0';
}
OS << DescStr;
for (uint64_t I = DescStr.size(); I < alignTo(DescStr.size(), 4); ++I) {
OS << '\0';
}
return OS.str();
}
}
void RewriteInstance::addBoltInfoSection() {
std::string DescStr;
raw_string_ostream DescOS(DescStr);
@ -4107,8 +4091,9 @@ void RewriteInstance::addBoltInfoSection() {
}
DescOS.flush();
// Encode as GNU GOLD VERSION so it is easily printable by 'readelf -n'
const auto BoltInfo =
encodeELFNote("GNU", DescStr, 4 /*NT_GNU_GOLD_VERSION*/);
BinarySection::encodeELFNote("GNU", DescStr, 4 /*NT_GNU_GOLD_VERSION*/);
BC->registerOrUpdateNoteSection(".note.bolt_info", copyByteArray(BoltInfo),
BoltInfo.size(),
/*Alignment=*/1,
@ -4129,7 +4114,8 @@ void RewriteInstance::encodeBATSection() {
BAT->write(DescOS);
DescOS.flush();
const auto BoltInfo = encodeELFNote("BOLT", DescStr, 0x1);
const auto BoltInfo =
BinarySection::encodeELFNote("BOLT", DescStr, BinarySection::NT_BOLT_BAT);
BC->registerOrUpdateNoteSection(BoltAddressTranslation::SECTION_NAME,
copyByteArray(BoltInfo), BoltInfo.size(),
/*Alignment=*/1,