Bug 1654413 - Add add_marker Rust profiler API and serialize the marker to the buffer r=emilio,gerald

This only adds the API and then adds the profiler payload to the buffer. The
deserialization and streaming will happen in the next patch.

Differential Revision: https://phabricator.services.mozilla.com/D124026

Cargo.lock

@@ -1781,9 +1781,11 @@ dependencies = [
 name = "gecko-profiler"
 version = "0.1.0"
 dependencies = [
+ "bincode",
  "bindgen",
  "lazy_static",
  "profiler-macros",
+ "serde",
 ]
 
 [[package]]

mozglue/baseprofiler/public/BaseProfilerMarkersDetail.h

@@ -109,7 +109,8 @@ struct StreamFunctionTypeHelper<R(baseprofiler::SpliceableJSONWriter&, As...)> {
     // Note that options are first after the entry kind, because they contain
     // the thread id, which is handled first to filter markers by threads.
     return aBuffer.PutObjects(ProfileBufferEntryKind::Marker, aOptions, aName,
-                              aCategory, aDeserializerTag, aAs...);
+                              aCategory, aDeserializerTag,
+                              MarkerPayloadType::Cpp, aAs...);
   }
 };
 
@@ -304,6 +305,9 @@ template <typename StackCallback>
   //   ProfileBufferEntry::Kind::Marker, <- already read by caller
   //   options,                          <- next location in entries
   //   name,
+  //   category,
+  //   deserializer tag,
+  //   payload type (cpp or rust)
   //   payload
   const MarkerOptions options = aEntryReader.ReadObject<MarkerOptions>();
   if (aThreadIdOrUnspecified.IsSpecified() &&
@@ -357,12 +361,30 @@ template <typename StackCallback>
           aWriter.EndObject();
         }
 
+        auto payloadType = static_cast<mozilla::MarkerPayloadType>(
+            aEntryReader
+                .ReadObject<mozilla::MarkerPayloadTypeUnderlyingType>());
+
         // Stream the payload, including the type.
-        mozilla::base_profiler_markers_detail::Streaming::MarkerDataDeserializer
-            deserializer = mozilla::base_profiler_markers_detail::Streaming::
+        switch (payloadType) {
+          case mozilla::MarkerPayloadType::Cpp: {
+            mozilla::base_profiler_markers_detail::Streaming::
+                MarkerDataDeserializer deserializer =
+                    mozilla::base_profiler_markers_detail::Streaming::
                         DeserializerForTag(tag);
+
             MOZ_RELEASE_ASSERT(deserializer);
             deserializer(aEntryReader, aWriter);
+            break;
+          }
+          case mozilla::MarkerPayloadType::Rust:
+            // FIXME: Not implemented yet. It will be implemented in the
+            // following patch.
+            break;
+          default:
+            MOZ_ASSERT_UNREACHABLE("Unknown payload type.");
+            break;
+        }
       }
       aWriter.EndObject();
     }

mozglue/baseprofiler/public/ProfileBufferEntryKinds.h

@@ -97,6 +97,12 @@ enum class ProfileBufferEntryKind : ProfileBufferEntryKindUnderlyingType {
   MODERN_LIMIT
 };
 
+using MarkerPayloadTypeUnderlyingType = uint8_t;
+enum class MarkerPayloadType : MarkerPayloadTypeUnderlyingType {
+  Cpp,
+  Rust,
+};
+
 }  // namespace mozilla
 
 #endif  // ProfileBufferEntryKinds_h

tools/profiler/core/ProfilerBindings.cpp

@@ -284,3 +284,64 @@ void gecko_profiler_add_marker_text(
       mozilla::ProfilerString8View(aText, aTextLength));
 #endif
 }
+
+void gecko_profiler_add_marker(
+    const char* aName, size_t aNameLength,
+    mozilla::baseprofiler::ProfilingCategoryPair aCategoryPair,
+    mozilla::MarkerTiming* aMarkerTiming,
+    mozilla::StackCaptureOptions aStackCaptureOptions, uint8_t aMarkerTag,
+    const uint8_t* aPayload, size_t aPayloadSize) {
+#ifdef MOZ_GECKO_PROFILER
+  // Copy the marker timing and create the marker option.
+  mozilla::MarkerOptions markerOptions(
+      std::move(*aMarkerTiming),
+      mozilla::MarkerStack::WithCaptureOptions(aStackCaptureOptions));
+
+  // Currently it's not possible to add a threadId option, but we will
+  // have it soon.
+  if (markerOptions.ThreadId().IsUnspecified()) {
+    // If yet unspecified, set thread to this thread where the marker is added.
+    markerOptions.Set(mozilla::MarkerThreadId::CurrentThread());
+  }
+
+  auto& buffer = profiler_markers_detail::CachedCoreBuffer();
+  mozilla::Span payload(aPayload, aPayloadSize);
+
+  mozilla::StackCaptureOptions captureOptions =
+      markerOptions.Stack().CaptureOptions();
+  if (captureOptions != mozilla::StackCaptureOptions::NoStack) {
+    // A capture was requested, let's attempt to do it here&now. This avoids a
+    // lot of allocations that would be necessary if capturing a backtrace
+    // separately.
+    // TODO use a local on-stack byte buffer to remove last allocation.
+    // TODO reduce internal profiler stack levels, see bug 1659872.
+    mozilla::ProfileBufferChunkManagerSingle chunkManager(
+        mozilla::ProfileBufferChunkManager::scExpectedMaximumStackSize);
+    mozilla::ProfileChunkedBuffer chunkedBuffer(
+        mozilla::ProfileChunkedBuffer::ThreadSafety::WithoutMutex,
+        chunkManager);
+    markerOptions.StackRef().UseRequestedBacktrace(
+        profiler_capture_backtrace_into(chunkedBuffer, captureOptions)
+            ? &chunkedBuffer
+            : nullptr);
+
+    // This call must be made from here, while chunkedBuffer is in scope.
+    buffer.PutObjects(
+        mozilla::ProfileBufferEntryKind::Marker, markerOptions,
+        mozilla::ProfilerString8View(aName, aNameLength),
+        mozilla::MarkerCategory{aCategoryPair},
+        mozilla::base_profiler_markers_detail::Streaming::DeserializerTag(
+            aMarkerTag),
+        mozilla::MarkerPayloadType::Rust, payload);
+    return;
+  }
+
+  buffer.PutObjects(
+      mozilla::ProfileBufferEntryKind::Marker, markerOptions,
+      mozilla::ProfilerString8View(aName, aNameLength),
+      mozilla::MarkerCategory{aCategoryPair},
+      mozilla::base_profiler_markers_detail::Streaming::DeserializerTag(
+          aMarkerTag),
+      mozilla::MarkerPayloadType::Rust, payload);
+#endif
+}

tools/profiler/public/ProfilerBindings.h

@@ -134,6 +134,12 @@ void gecko_profiler_add_marker_text(
     mozilla::MarkerTiming* aMarkerTiming,
     mozilla::StackCaptureOptions aStackCaptureOptions, const char* aText,
     size_t aTextLength);
+void gecko_profiler_add_marker(
+    const char* aName, size_t aNameLength,
+    mozilla::baseprofiler::ProfilingCategoryPair aCategoryPair,
+    mozilla::MarkerTiming* aMarkerTiming,
+    mozilla::StackCaptureOptions aStackCaptureOptions, uint8_t aMarkerTag,
+    const uint8_t* aPayload, size_t aPayloadSize);
 
 }  // extern "C"
 

tools/profiler/rust-api/Cargo.toml

@@ -6,6 +6,9 @@ edition = "2018"
 
 [dependencies]
 profiler-macros = { path = "./macros" }
+lazy_static = "1"
+serde = { version = "1.0", features = ["derive"] }
+bincode = "1"
 
 [build-dependencies]
 lazy_static = "1"

tools/profiler/rust-api/src/lib.rs

@@ -3,6 +3,10 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 ///! Profiler Rust API
+
+#[macro_use]
+extern crate lazy_static;
+
 pub mod gecko_bindings;
 mod json_writer;
 mod label;
@@ -21,3 +25,5 @@ pub use profiler_macros::gecko_profiler_fn_label;
 pub use profiler_state::*;
 pub use thread::*;
 pub use time::*;
+
+pub use serde::{Deserialize, Serialize};

tools/profiler/rust-api/src/marker/deserializer_tags_state.rs

@@ -0,0 +1,91 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+use crate::json_writer::JSONWriter;
+use crate::marker::schema::MarkerSchema;
+use crate::marker::{transmute_and_stream, ProfilerMarker};
+use std::collections::HashMap;
+use std::sync::RwLock;
+
+lazy_static! {
+    static ref DESERIALIZER_TAGS_STATE: RwLock<DeserializerTagsState> =
+        RwLock::new(DeserializerTagsState::new());
+}
+
+/// A state that keeps track of each marker types and their deserializer tags.
+/// They are added during the marker insertion and read during the marker serialization.
+pub struct DeserializerTagsState {
+    /// C++ side accepts only u8 values, but we only know usize values as the
+    /// unique marker type values. So, we need to keep track of each
+    /// "marker tag -> deserializer tag" conversions to directly get the
+    /// deserializer tags of the already added marker types.
+    pub marker_tag_to_deserializer_tag: HashMap<usize, u8>,
+    /// Vector of marker type functions.
+    /// 1-based, i.e.: [0] -> tag 1. Elements are pushed to the end of the vector
+    /// whenever a new marker type is used in a Firefox session; the content is
+    /// kept between profiler runs in that session. On the C++ side, we have the
+    /// same algorithm (althought it's a sized array). See `sMarkerTypeFunctions1Based`.
+    pub marker_type_functions_1_based: Vec<MarkerTypeFunctions>,
+}
+
+/// Functions that will be stored per marker type, so we can serialize the marker
+/// schema and stream the marker payload for a specific type.
+pub struct MarkerTypeFunctions {
+    /// A function that returns the name of the marker type.
+    pub marker_type_name_fn: fn() -> &'static str,
+    /// A function that returns a `MarkerSchema`, which contains all the
+    /// information needed to stream the display schema associated with a
+    /// marker type.
+    pub marker_type_display_fn: fn() -> MarkerSchema,
+    /// A function that can read a serialized payload from bytes and streams it
+    /// as JSON object properties.
+    pub transmute_and_stream_fn:
+        unsafe fn(payload: *const u8, payload_size: usize, json_writer: &mut JSONWriter),
+}
+
+impl DeserializerTagsState {
+    fn new() -> Self {
+        DeserializerTagsState {
+            marker_tag_to_deserializer_tag: HashMap::new(),
+            marker_type_functions_1_based: vec![],
+        }
+    }
+}
+
+/// Get or insert the deserializer tag for each marker type. The tag storage
+/// is limited to 255 marker types. This is the same with the C++ side. It's
+/// unlikely to reach to this limit, but if that's the case, C++ side needs
+/// to change the uint8_t type for the deserializer tag as well.
+pub fn get_or_insert_deserializer_tag<T>() -> u8
+where
+    T: ProfilerMarker,
+{
+    let unique_marker_tag = &T::marker_type_name as *const _ as usize;
+    let mut state = DESERIALIZER_TAGS_STATE.write().unwrap();
+
+    match state.marker_tag_to_deserializer_tag.get(&unique_marker_tag) {
+        None => {
+            // It's impossible to have length more than u8.
+            let deserializer_tag = state.marker_type_functions_1_based.len() as u8 + 1;
+            debug_assert!(
+                deserializer_tag < 250,
+                "Too many rust marker payload types! Please consider increasing the profiler \
+                 buffer tag size."
+            );
+
+            state
+                .marker_tag_to_deserializer_tag
+                .insert(unique_marker_tag, deserializer_tag);
+            state
+                .marker_type_functions_1_based
+                .push(MarkerTypeFunctions {
+                    marker_type_name_fn: T::marker_type_name,
+                    marker_type_display_fn: T::marker_type_display,
+                    transmute_and_stream_fn: transmute_and_stream::<T>,
+                });
+            deserializer_tag
+        }
+        Some(deserializer_tag) => *deserializer_tag,
+    }
+}

tools/profiler/rust-api/src/marker/mod.rs

@@ -64,13 +64,68 @@
 //! );
 //! ```
 //!
-//! TODO: Explain the marker API once we have them in the following patches.
+//! ### Marker with a more complex payload and different visualization in the profiler front-end.
+//!
+//! [`add_marker`] is the most advanced API that you can use to add different types
+//! of values as data to your marker and customize the visualization of that marker
+//! in the profiler front-end (profiler.firefox.com).
+//!
+//! To be able to add a a marker, first you need to create your marker payload
+//! struct in your codebase and implement the [`ProfilerMarker`] trait like this:
+//!
+//! ```
+//! #[derive(Serialize, Deserialize, Debug)]
+//! pub struct TestMarker {
+//!     a: u32,
+//!     b: String,
+//! }
+//!
+//! // Please see the documentation of [`ProfilerMarker`].
+//! impl gecko_profiler::ProfilerMarker for TestMarker {
+//!     fn marker_type_name() -> &'static str {
+//!         "marker type from rust"
+//!     }
+//!     fn marker_type_display() -> gecko_profiler::MarkerSchema {
+//!         use gecko_profiler::schema::*;
+//!         let mut schema = MarkerSchema::new(&[Location::MarkerChart]);
+//!         schema.set_chart_label("Name: {marker.name}");
+//!         schema.set_tooltip_label("{marker.data.a}");
+//!         schema.add_key_label_format("a", "A Value", Format::Integer);
+//!         schema.add_key_label_format("b", "B Value", Format::String);
+//!         schema
+//!     }
+//!     fn stream_json_marker_data(&self, json_writer: &mut gecko_profiler::JSONWriter) {
+//!         json_writer.int_property("a", self.a.into());
+//!         json_writer.string_property("b", &self.b);
+//!     }
+//! }
+//! ```
+//!
+//! Once you've created this payload and implemented the [`ProfilerMarker`], you
+//! can now add this marker in the code that you would like to measure. E.g.:
+//!
+//! ```
+//! gecko_profiler::add_marker(
+//!     // Name of the marker as a string.
+//!     "Marker Name",
+//!     // Category with an optional sub-category.
+//!     gecko_profiler_category!(Graphics, DisplayListBuilding),
+//!     // MarkerOptions that keeps options like marker timing and marker stack.
+//!     Default::default(),
+//!     // Marker payload.
+//!     TestMarker {a: 12, b: "hello".to_owned()},
+//! );
+//! ```
 
+pub(crate) mod deserializer_tags_state;
 pub mod options;
 pub mod schema;
 
 use crate::gecko_bindings::{bindings, profiling_categories::ProfilingCategoryPair};
+use crate::json_writer::JSONWriter;
+use crate::marker::deserializer_tags_state::get_or_insert_deserializer_tag;
 use crate::marker::options::MarkerOptions;
+use serde::{de::DeserializeOwned, Serialize};
 use std::os::raw::c_char;
 
 /// Marker API to add a new simple marker without any payload.
@@ -117,3 +172,77 @@ pub fn add_text_marker(
         )
     }
 }
+
+/// Trait that every profiler marker payload struct needs to implement.
+/// This will tell the profiler back-end how to serialize it as json and
+/// the front-end how to display the marker.
+/// Please also see the documentation here:
+/// https://firefox-source-docs.mozilla.org/tools/profiler/markers-guide.html#how-to-define-new-marker-types
+///
+/// - `marker_type_name`: Returns a static string as the marker type name. This
+/// should be unique and it is used to keep track of the type of markers in the
+/// profiler storage, and to identify them uniquely on the profiler front-end.
+/// - `marker_type_display`: Where and how to display the marker and its data.
+/// Returns a `MarkerSchema` object which will be forwarded to the profiler
+/// front-end.
+/// - `stream_json_marker_data`: Data specific to this marker type should be
+/// serialized to JSON for the profiler front-end. All the common marker data
+/// like marker name, category, timing will be serialized automatically. But
+/// marker specific data should be serialized here.
+pub trait ProfilerMarker: Serialize + DeserializeOwned {
+    /// A static method that returns the name of the marker type.
+    fn marker_type_name() -> &'static str;
+    /// A static method that returns a `MarkerSchema`, which contains all the
+    /// information needed to stream the display schema associated with a
+    /// marker type.
+    fn marker_type_display() -> schema::MarkerSchema;
+    /// A method that streams the marker payload data as JSON object properties.
+    /// Please see the [JSONWriter] struct to see its methods.
+    fn stream_json_marker_data(&self, json_writer: &mut JSONWriter);
+}
+
+/// A function that deserializes the marker payload and streams it to the JSON.
+unsafe fn transmute_and_stream<T>(
+    payload: *const u8,
+    payload_size: usize,
+    json_writer: &mut JSONWriter,
+) where
+    T: ProfilerMarker,
+{
+    let payload_slice = std::slice::from_raw_parts(payload, payload_size);
+    let payload: T = bincode::deserialize(&payload_slice).unwrap();
+    payload.stream_json_marker_data(json_writer);
+}
+
+/// Main marker API to add a new marker to profiler buffer.
+/// Please see the module documentation on how to add a marker with this API.
+pub fn add_marker<T>(
+    name: &str,
+    category: ProfilingCategoryPair,
+    mut options: MarkerOptions,
+    payload: T,
+) where
+    T: ProfilerMarker,
+{
+    if !crate::profiler_state::can_accept_markers() {
+        // Nothing to do.
+        return;
+    }
+
+    let encoded_payload: Vec<u8> = bincode::serialize(&payload).unwrap();
+    let payload_size = encoded_payload.len();
+    let maker_tag = get_or_insert_deserializer_tag::<T>();
+
+    unsafe {
+        bindings::gecko_profiler_add_marker(
+            name.as_ptr() as *const c_char,
+            name.len(),
+            category.to_cpp_enum_value(),
+            options.timing.0.as_mut_ptr(),
+            options.stack,
+            maker_tag,
+            encoded_payload.as_ptr(),
+            payload_size,
+        )
+    }
+}