Bug 1457481 - Add nsIProfiler.GetSymbolTable and a profiler/rust-helper crate which implements it for ELF binaries. r=njn,jrmuizel

r?njn for the profiler parts
r?jrmuizel for the ELF parsing parts

Depends on D7020

Differential Revision: https://phabricator.services.mozilla.com/D7021

--HG--
extra : moz-landing-system : lando

toolkit/library/gtest/rust/Cargo.toml

@@ -15,6 +15,8 @@ gecko_debug = ["gkrust-shared/gecko_debug"]
 simd-accel = ["gkrust-shared/simd-accel"]
 moz_memory = ["gkrust-shared/moz_memory"]
 spidermonkey_rust = ["gkrust-shared/spidermonkey_rust"]
+gecko_profiler = ["gkrust-shared/gecko_profiler"]
+gecko_profiler_parse_elf = ["gkrust-shared/gecko_profiler_parse_elf"]
 
 [dependencies]
 bench-collections-gtest = { path = "../../../../xpcom/rust/gtest/bench-collections" }

toolkit/library/rust/Cargo.toml

@@ -15,6 +15,8 @@ gecko_debug = ["gkrust-shared/gecko_debug"]
 simd-accel = ["gkrust-shared/simd-accel"]
 moz_memory = ["gkrust-shared/moz_memory"]
 spidermonkey_rust = ["gkrust-shared/spidermonkey_rust"]
+gecko_profiler = ["gkrust-shared/gecko_profiler"]
+gecko_profiler_parse_elf = ["gkrust-shared/gecko_profiler_parse_elf"]
 
 [dependencies]
 gkrust-shared = { path = "shared" }

toolkit/library/rust/gkrust-features.mozbuild

@@ -28,3 +28,9 @@ if CONFIG['MOZ_MEMORY']:
 
 if CONFIG['ENABLE_WASM_CRANELIFT']:
     gkrust_features += ['spidermonkey_rust']
+
+if CONFIG['MOZ_GECKO_PROFILER']:
+    gkrust_features += ['gecko_profiler']
+
+if CONFIG['MOZ_GECKO_PROFILER_PARSE_ELF']:
+    gkrust_features += ['gecko_profiler_parse_elf']

toolkit/library/rust/shared/Cargo.toml

@@ -13,6 +13,7 @@ nserror = { path = "../../../../xpcom/rust/nserror" }
 netwerk_helper = { path = "../../../../netwerk/base/rust-helper" }
 xpcom = { path = "../../../../xpcom/rust/xpcom" }
 prefs_parser = { path = "../../../../modules/libpref/parser" }
+profiler_helper = { path = "../../../../tools/profiler/rust-helper", optional = true }
 mozurl = { path = "../../../../netwerk/base/mozurl" }
 webrender_bindings = { path = "../../../../gfx/webrender_bindings", optional = true }
 cubeb-pulse = { path = "../../../../media/libcubeb/cubeb-pulse-rs", optional = true, features=["pulse-dlopen"] }
@@ -46,6 +47,8 @@ gecko_debug = ["geckoservo/gecko_debug", "nsstring/gecko_debug"]
 simd-accel = ["encoding_c/simd-accel", "encoding_glue/simd-accel"]
 moz_memory = ["mp4parse_capi/mp4parse_fallible"]
 spidermonkey_rust = ["jsrust_shared"]
+gecko_profiler = ["profiler_helper"]
+gecko_profiler_parse_elf = ["profiler_helper/parse_elf"]
 
 [lib]
 path = "lib.rs"

toolkit/library/rust/shared/lib.rs

@@ -15,6 +15,8 @@ extern crate nserror;
 extern crate xpcom;
 extern crate netwerk_helper;
 extern crate prefs_parser;
+#[cfg(feature = "gecko_profiler")]
+extern crate profiler_helper;
 extern crate mozurl;
 #[cfg(feature = "quantum_render")]
 extern crate webrender_bindings;

tools/profiler/gecko/nsIProfiler.idl

@@ -122,6 +122,31 @@ interface nsIProfiler : nsISupports
   [implicit_jscontext]
   readonly attribute jsval sharedLibraries;
 
+  /**
+   * Returns a promise that resolves to a SymbolTableAsTuple for the binary at
+   * the given path.
+   *
+   * SymbolTable as tuple: [addrs, index, buffer]
+   * Contains a symbol table, which can be used to map addresses to strings.
+   *
+   * The first element of this tuple, commonly named "addrs", is a sorted array of
+   * symbol addresses, as library-relative offsets in bytes, in ascending order.
+   * The third element of this tuple, commonly named "buffer", is a buffer of
+   * bytes that contains all strings from this symbol table, in the order of the
+   * addresses they correspond to, in utf-8 encoded form, all concatenated
+   * together.
+   * The second element of this tuple, commonly named "index", contains positions
+   * into "buffer". For every address, that position is where the string for that
+   * address starts in the buffer.
+   * index.length == addrs.length + 1.
+   * index[addrs.length] is the end position of the last string in the buffer.
+   *
+   * The string for the address addrs[i] is
+   * (new TextDecoder()).decode(buffer.subarray(index[i], index[i + 1]))
+   */
+  [implicit_jscontext]
+  nsISupports getSymbolTable(in ACString aDebugPath, in ACString aBreakpadID);
+
   /**
    * Dump the collected profile to a file.
    */

tools/profiler/gecko/nsProfiler.cpp

@@ -24,6 +24,8 @@
 #include "mozilla/ErrorResult.h"
 #include "mozilla/dom/Promise.h"
 #include "mozilla/dom/TypedArray.h"
+#include "mozilla/Move.h"
+#include "mozilla/SystemGroup.h"
 #include "nsLocalFile.h"
 #include "nsThreadUtils.h"
 #include "ProfilerParent.h"
@@ -35,6 +37,14 @@ using dom::AutoJSAPI;
 using dom::Promise;
 using std::string;
 
+extern "C" {
+  // This function is defined in the profiler rust module at tools/profiler/rust-helper.
+  // nsProfiler::SymbolTable and CompactSymbolTable have identical memory layout.
+  bool profiler_get_symbol_table(const char* debug_path,
+                                 const char* breakpad_id,
+                                 nsProfiler::SymbolTable* symbol_table);
+}
+
 NS_IMPL_ISUPPORTS(nsProfiler, nsIProfiler)
 
 nsProfiler::nsProfiler()
@@ -51,6 +61,9 @@ nsProfiler::~nsProfiler()
     observerService->RemoveObserver(this, "chrome-document-global-created");
     observerService->RemoveObserver(this, "last-pb-context-exited");
   }
+  if (mSymbolTableThread) {
+    mSymbolTableThread->Shutdown();
+  }
 }
 
 nsresult
@@ -394,6 +407,73 @@ nsProfiler::DumpProfileToFileAsync(const nsACString& aFilename,
   return NS_OK;
 }
 
+NS_IMETHODIMP
+nsProfiler::GetSymbolTable(const nsACString& aDebugPath,
+                           const nsACString& aBreakpadID,
+                           JSContext* aCx,
+                           nsISupports** aPromise)
+{
+  MOZ_ASSERT(NS_IsMainThread());
+
+  if (NS_WARN_IF(!aCx)) {
+    return NS_ERROR_FAILURE;
+  }
+
+  nsIGlobalObject* globalObject =
+    xpc::NativeGlobal(JS::CurrentGlobalOrNull(aCx));
+
+  if (NS_WARN_IF(!globalObject)) {
+    return NS_ERROR_FAILURE;
+  }
+
+  ErrorResult result;
+  RefPtr<Promise> promise = Promise::Create(globalObject, result);
+  if (NS_WARN_IF(result.Failed())) {
+    return result.StealNSResult();
+  }
+
+  GetSymbolTableMozPromise(aDebugPath, aBreakpadID)->Then(
+    GetMainThreadSerialEventTarget(), __func__,
+    [promise](const SymbolTable& aSymbolTable) {
+      AutoJSAPI jsapi;
+      if (NS_WARN_IF(!jsapi.Init(promise->GlobalJSObject()))) {
+        // We're really hosed if we can't get a JS context for some reason.
+        promise->MaybeReject(NS_ERROR_DOM_UNKNOWN_ERR);
+        return;
+      }
+
+      JSContext* cx = jsapi.cx();
+
+      JS::RootedObject addrsArray(
+        cx,
+        dom::Uint32Array::Create(
+          cx, aSymbolTable.mAddrs.Length(), aSymbolTable.mAddrs.Elements()));
+      JS::RootedObject indexArray(
+        cx,
+        dom::Uint32Array::Create(
+          cx, aSymbolTable.mIndex.Length(), aSymbolTable.mIndex.Elements()));
+      JS::RootedObject bufferArray(
+        cx,
+        dom::Uint8Array::Create(
+          cx, aSymbolTable.mBuffer.Length(), aSymbolTable.mBuffer.Elements()));
+
+      if (addrsArray && indexArray && bufferArray) {
+        JS::RootedObject tuple(cx, JS_NewArrayObject(cx, 3));
+        JS_SetElement(cx, tuple, 0, addrsArray);
+        JS_SetElement(cx, tuple, 1, indexArray);
+        JS_SetElement(cx, tuple, 2, bufferArray);
+        promise->MaybeResolve(tuple);
+      } else {
+        promise->MaybeReject(NS_ERROR_FAILURE);
+      }
+    },
+    [promise](nsresult aRv) {
+      promise->MaybeReject(aRv);
+    });
+
+  promise.forget(aPromise);
+  return NS_OK;
+}
 
 NS_IMETHODIMP
 nsProfiler::GetElapsedTime(double* aElapsedTime)
@@ -629,6 +709,48 @@ nsProfiler::StartGathering(double aSinceTime)
   return promise;
 }
 
+RefPtr<nsProfiler::SymbolTablePromise>
+nsProfiler::GetSymbolTableMozPromise(const nsACString& aDebugPath,
+                                     const nsACString& aBreakpadID)
+{
+  MozPromiseHolder<SymbolTablePromise> promiseHolder;
+  RefPtr<SymbolTablePromise> promise = promiseHolder.Ensure(__func__);
+
+  if (!mSymbolTableThread) {
+    nsresult rv =
+      NS_NewNamedThread("ProfSymbolTable", getter_AddRefs(mSymbolTableThread));
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      promiseHolder.Reject(NS_ERROR_FAILURE, __func__);
+      return promise;
+    }
+  }
+
+  mSymbolTableThread->Dispatch(NS_NewRunnableFunction(
+    "nsProfiler::GetSymbolTableMozPromise runnable on ProfSymbolTable thread",
+    [promiseHolder = std::move(promiseHolder),
+     debugPath = nsCString(aDebugPath),
+     breakpadID = nsCString(aBreakpadID)]() mutable {
+      AUTO_PROFILER_LABEL_DYNAMIC_NSCSTRING("profiler_get_symbol_table", OTHER, debugPath);
+      SymbolTable symbolTable;
+      bool succeeded = profiler_get_symbol_table(debugPath.get(), breakpadID.get(), &symbolTable);
+      SystemGroup::Dispatch(TaskCategory::Other, NS_NewRunnableFunction(
+        "nsProfiler::GetSymbolTableMozPromise result on main thread",
+        [promiseHolder = std::move(promiseHolder),
+         symbolTable = std::move(symbolTable),
+         succeeded]() mutable {
+          if (succeeded) {
+            promiseHolder.Resolve(std::move(symbolTable), __func__);
+          } else {
+            promiseHolder.Reject(NS_ERROR_FAILURE, __func__);
+          }
+        }
+      ));
+    }
+  ));
+
+  return promise;
+}
+
 void
 nsProfiler::FinishGathering()
 {

tools/profiler/gecko/nsProfiler.h

@@ -36,10 +36,23 @@ public:
 
   void GatheredOOPProfile(const nsACString& aProfile);
 
+  // This SymbolTable struct, and the CompactSymbolTable struct in the
+  // profiler rust module, have the exact same memory layout.
+  // nsTArray and ThinVec are FFI-compatible, because the thin-vec crate is
+  // being compiled with the "gecko-ffi" feature enabled.
+  struct SymbolTable {
+    SymbolTable() = default;
+    SymbolTable(SymbolTable&& aOther) = default;
+
+    nsTArray<uint32_t> mAddrs;
+    nsTArray<uint32_t> mIndex;
+    nsTArray<uint8_t> mBuffer;
+  };
 private:
   ~nsProfiler();
 
   typedef mozilla::MozPromise<nsCString, nsresult, false> GatheringPromise;
+  typedef mozilla::MozPromise<SymbolTable, nsresult, true> SymbolTablePromise;
 
   RefPtr<GatheringPromise> StartGathering(double aSinceTime);
   void FinishGathering();
@@ -47,6 +60,9 @@ private:
 
   void ClearExpiredExitProfiles();
 
+  RefPtr<SymbolTablePromise> GetSymbolTableMozPromise(const nsACString& aDebugPath,
+                                                      const nsACString& aBreakpadID);
+
   bool mLockedForPrivateBrowsing;
 
   struct ExitProfile {
@@ -57,6 +73,7 @@ private:
   // These fields are all related to profile gathering.
   nsTArray<ExitProfile> mExitProfiles;
   mozilla::Maybe<mozilla::MozPromiseHolder<GatheringPromise>> mPromiseHolder;
+  nsCOMPtr<nsIThread> mSymbolTableThread;
   mozilla::Maybe<SpliceableChunkedJSONWriter> mWriter;
   uint32_t mPendingProfiles;
   bool mGathering;

tools/profiler/rust-helper/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+name = "profiler_helper"
+version = "0.1.0"
+authors = ["Markus Stange <mstange@themasta.com>"]
+
+[dependencies]
+memmap = "0.6.2"
+object = { version = "0.10.0", optional = true, default-features = false }
+
+[dependencies.thin-vec]
+version = "0.1.0"
+features = ["gecko-ffi"]
+
+[features]
+parse_elf = ["object"]

tools/profiler/rust-helper/src/compact_symbol_table.rs

@@ -0,0 +1,36 @@
+use std::collections::HashMap;
+use thin_vec::ThinVec;
+
+#[repr(C)]
+pub struct CompactSymbolTable {
+    pub addr: ThinVec<u32>,
+    pub index: ThinVec<u32>,
+    pub buffer: ThinVec<u8>,
+}
+
+impl CompactSymbolTable {
+    pub fn new() -> Self {
+        Self {
+            addr: ThinVec::new(),
+            index: ThinVec::new(),
+            buffer: ThinVec::new(),
+        }
+    }
+
+    pub fn from_map(map: HashMap<u32, &str>) -> Self {
+        let mut table = Self::new();
+        let mut entries: Vec<_> = map.into_iter().collect();
+        entries.sort_by_key(|&(addr, _)| addr);
+        for (addr, name) in entries {
+            table.addr.push(addr);
+            table.index.push(table.buffer.len() as u32);
+            table.add_name(name);
+        }
+        table.index.push(table.buffer.len() as u32);
+        table
+    }
+
+    fn add_name(&mut self, name: &str) {
+        self.buffer.extend_from_slice(name.as_bytes());
+    }
+}

tools/profiler/rust-helper/src/elf.rs

@@ -0,0 +1,24 @@
+use compact_symbol_table::CompactSymbolTable;
+use object::{ElfFile, Object, SymbolKind, Uuid};
+use std::collections::HashMap;
+
+fn get_symbol_map<'a, 'b, T>(object_file: &'b T) -> HashMap<u32, &'a str>
+where
+  T: Object<'a, 'b>,
+{
+  object_file
+    .dynamic_symbols()
+    .chain(object_file.symbols())
+    .filter(|symbol| symbol.kind() == SymbolKind::Text)
+    .filter_map(|symbol| symbol.name().map(|name| (symbol.address() as u32, name)))
+    .collect()
+}
+
+pub fn get_compact_symbol_table(buffer: &[u8], breakpad_id: &str) -> Option<CompactSymbolTable> {
+  let elf_file = ElfFile::parse(buffer).ok()?;
+  let elf_id = Uuid::from_bytes(elf_file.build_id()?).ok()?;
+  if format!("{:X}0", elf_id.simple()) != breakpad_id {
+    return None;
+  }
+  return Some(CompactSymbolTable::from_map(get_symbol_map(&elf_file)));
+}

tools/profiler/rust-helper/src/lib.rs

@@ -0,0 +1,55 @@
+extern crate memmap;
+extern crate thin_vec;
+
+#[cfg(feature = "parse_elf")]
+extern crate object;
+
+mod compact_symbol_table;
+
+#[cfg(feature = "parse_elf")]
+mod elf;
+
+#[cfg(feature = "parse_elf")]
+use memmap::MmapOptions;
+#[cfg(feature = "parse_elf")]
+use std::fs::File;
+
+use std::ffi::CStr;
+use std::os::raw::c_char;
+use compact_symbol_table::CompactSymbolTable;
+
+#[cfg(feature = "parse_elf")]
+pub fn get_compact_symbol_table_from_file(
+    debug_path: &str,
+    breakpad_id: &str,
+) -> Option<CompactSymbolTable> {
+    let file = File::open(debug_path).ok()?;
+    let buffer = unsafe { MmapOptions::new().map(&file).ok()? };
+    elf::get_compact_symbol_table(&buffer, breakpad_id)
+}
+
+#[cfg(not(feature = "parse_elf"))]
+pub fn get_compact_symbol_table_from_file(
+    _debug_path: &str,
+    _breakpad_id: &str,
+) -> Option<CompactSymbolTable> {
+    None
+}
+
+#[no_mangle]
+pub extern "C" fn profiler_get_symbol_table(
+    debug_path: *const c_char,
+    breakpad_id: *const c_char,
+    symbol_table: &mut CompactSymbolTable,
+) -> bool {
+    let debug_path = unsafe { CStr::from_ptr(debug_path).to_string_lossy() };
+    let breakpad_id = unsafe { CStr::from_ptr(breakpad_id).to_string_lossy() };
+
+    match get_compact_symbol_table_from_file(&debug_path, &breakpad_id) {
+        Some(mut st) => {
+            std::mem::swap(symbol_table, &mut st);
+            true
+        }
+        None => false,
+    }
+}