Bug 1654413 - Deserialize and stream the markers and marker schemas r=emilio,gerald

Now we can deserialize and stream everything to the JSON.

Differential Revision: https://phabricator.services.mozilla.com/D124027

mozglue/baseprofiler/core/ProfileBufferEntry.cpp

@@ -817,6 +817,11 @@ void ProfileBuffer::StreamMarkersToJSON(SpliceableJSONWriter& aWriter,
                 ProfilerBacktrace backtrace("", &aChunkedBuffer);
                 backtrace.StreamJSON(aWriter, TimeStamp::ProcessCreation(),
                                      aUniqueStacks);
+              },
+              // We don't have Rust markers in the mozglue.
+              [&](mozilla::base_profiler_markers_detail::Streaming::
+                      DeserializerTag) {
+                MOZ_ASSERT_UNREACHABLE("No Rust markers in mozglue.");
               });
     }
 

mozglue/baseprofiler/public/BaseProfilerMarkersDetail.h

@@ -295,12 +295,12 @@ ProfileBufferBlockIndex AddMarkerToBuffer(
       aBuffer, aName, aCategory, std::move(aOptions), aTs...);
 }
 
-template <typename StackCallback>
+template <typename StackCallback, typename RustMarkerCallback>
 [[nodiscard]] bool DeserializeAfterKindAndStream(
     ProfileBufferEntryReader& aEntryReader,
     baseprofiler::SpliceableJSONWriter& aWriter,
     baseprofiler::BaseProfilerThreadId aThreadIdOrUnspecified,
-    StackCallback&& aStackCallback) {
+    StackCallback&& aStackCallback, RustMarkerCallback&& aRustMarkerCallback) {
   // Each entry is made up of the following:
   //   ProfileBufferEntry::Kind::Marker, <- already read by caller
   //   options,                          <- next location in entries
@@ -378,8 +378,7 @@ template <typename StackCallback>
             break;
           }
           case mozilla::MarkerPayloadType::Rust:
-            // FIXME: Not implemented yet. It will be implemented in the
-            // following patch.
+            std::forward<RustMarkerCallback>(aRustMarkerCallback)(tag);
             break;
           default:
             MOZ_ASSERT_UNREACHABLE("Unknown payload type.");

mozglue/tests/TestBaseProfiler.cpp

@@ -4372,7 +4372,9 @@ void StreamMarkers(const mozilla::ProfileChunkedBuffer& aBuffer,
               mozilla::baseprofiler::BaseProfilerThreadId{},
               [&](mozilla::ProfileChunkedBuffer&) {
                 aWriter.StringElement("Real backtrace would be here");
-              });
+              },
+              [&](mozilla::base_profiler_markers_detail::Streaming::
+                      DeserializerTag) {});
       MOZ_RELEASE_ASSERT(success);
     });
   }

tools/profiler/core/ProfileBufferEntry.cpp

@@ -10,6 +10,7 @@
 #include "platform.h"
 #include "ProfileBuffer.h"
 #include "ProfilerBacktrace.h"
+#include "ProfilerRustBindings.h"
 
 #include "js/ProfilingFrameIterator.h"
 #include "jsapi.h"
@@ -1287,6 +1288,27 @@ void ProfileBuffer::StreamMarkersToJSON(SpliceableJSONWriter& aWriter,
               [&](ProfileChunkedBuffer& aChunkedBuffer) {
                 ProfilerBacktrace backtrace("", &aChunkedBuffer);
                 backtrace.StreamJSON(aWriter, aProcessStartTime, aUniqueStacks);
+              },
+              [&](mozilla::base_profiler_markers_detail::Streaming::
+                      DeserializerTag aTag) {
+                size_t payloadSize = aER.RemainingBytes();
+
+                ProfileBufferEntryReader::DoubleSpanOfConstBytes spans =
+                    aER.ReadSpans(payloadSize);
+                if (MOZ_LIKELY(spans.IsSingleSpan())) {
+                  // Only a single span, we can just refer to it directly
+                  // instead of copying it.
+                  profiler::ffi::gecko_profiler_serialize_marker_for_tag(
+                      aTag, spans.mFirstOrOnly.Elements(), payloadSize,
+                      &aWriter);
+                } else {
+                  // Two spans, we need to concatenate them by copying.
+                  uint8_t* payloadBuffer = new uint8_t[payloadSize];
+                  spans.CopyBytesTo(payloadBuffer);
+                  profiler::ffi::gecko_profiler_serialize_marker_for_tag(
+                      aTag, payloadBuffer, payloadSize, &aWriter);
+                  delete[] payloadBuffer;
+                }
               });
     }
 

tools/profiler/core/ProfilerBindings.cpp

@@ -212,6 +212,14 @@ void gecko_profiler_marker_schema_add_static_label_value(
 #endif
 }
 
+void gecko_profiler_marker_schema_stream(
+    mozilla::baseprofiler::SpliceableJSONWriter* aWriter, const char* aName,
+    size_t aNameLength, mozilla::MarkerSchema* aMarkerSchema) {
+#ifdef MOZ_GECKO_PROFILER
+  std::move(*aMarkerSchema).Stream(*aWriter, mozilla::Span(aName, aNameLength));
+#endif
+}
+
 void gecko_profiler_json_writer_int_property(
     mozilla::baseprofiler::SpliceableJSONWriter* aWriter, const char* aName,
     size_t aNameLength, int64_t aValue) {

tools/profiler/core/platform.cpp

@@ -38,6 +38,7 @@
 #include "ProfilerCodeAddressService.h"
 #include "ProfilerIOInterposeObserver.h"
 #include "ProfilerParent.h"
+#include "ProfilerRustBindings.h"
 #include "shared-libraries.h"
 #include "VTuneProfiler.h"
 
@@ -2474,6 +2475,9 @@ static void StreamMarkerSchema(SpliceableJSONWriter& aWriter) {
       markerTypeFunctions.mMarkerSchemaFunction().Stream(aWriter, name);
     }
   }
+
+  // Now stream the Rust marker schemas.
+  profiler::ffi::gecko_profiler_stream_marker_schemas(&aWriter);
 }
 
 // Some meta information that is better recorded before streaming the profile.

tools/profiler/moz.build

@@ -144,6 +144,7 @@ EXPORTS += [
     "public/GeckoProfiler.h",
     "public/ProfilerBindings.h",
     "public/ProfilerParent.h",
+    "public/ProfilerRustBindings.h",
 ]
 
 EXPORTS.mozilla += [
@@ -176,6 +177,14 @@ CONFIGURE_SUBST_FILES += [
     "rust-api/extra-bindgen-flags",
 ]
 
+
+if CONFIG["COMPILE_ENVIRONMENT"]:
+    CbindgenHeader("profiler_ffi_generated.h", inputs=["rust-api"])
+
+    EXPORTS.mozilla += [
+        "!profiler_ffi_generated.h",
+    ]
+
 FINAL_LIBRARY = "xul"
 
 if CONFIG["ENABLE_TESTS"]:

tools/profiler/public/ProfilerBindings.h

@@ -105,6 +105,11 @@ void gecko_profiler_marker_schema_add_static_label_value(
     mozilla::MarkerSchema* aSchema, const char* aLabel, size_t aLabelLength,
     const char* aValue, size_t aValueLength);
 
+// Stream MarkerSchema to SpliceableJSONWriter.
+void gecko_profiler_marker_schema_stream(
+    mozilla::baseprofiler::SpliceableJSONWriter* aWriter, const char* aName,
+    size_t aNameLength, mozilla::MarkerSchema* aMarkerSchema);
+
 // Various SpliceableJSONWriter methods to add properties.
 void gecko_profiler_json_writer_int_property(
     mozilla::baseprofiler::SpliceableJSONWriter* aWriter, const char* aName,

tools/profiler/public/ProfilerRustBindings.h

@@ -0,0 +1,12 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+#ifndef ProfilerRustBindings_h
+#define ProfilerRustBindings_h
+
+#include "mozilla/profiler_ffi_generated.h"
+
+// Add any non-generated support code here
+
+#endif  // ProfilerRustBindings_h

tools/profiler/rust-api/cbindgen.toml

@@ -0,0 +1,15 @@
+header = """/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */"""
+autogen_warning = """/* DO NOT MODIFY THIS MANUALLY! This file was generated using cbindgen. See RunCbindgen.py */
+#ifndef ProfilerRustBindings_h
+#error "Don't include this file directly, instead include ProfilerRustBindings.h"
+#endif
+"""
+include_version = true
+braces = "SameLine"
+line_length = 100
+tab_width = 2
+language = "C++"
+# Put FFI calls in the `mozilla::profiler::ffi` namespace.
+namespaces = ["mozilla", "profiler", "ffi"]

tools/profiler/rust-api/src/gecko_bindings/glue.rs

@@ -0,0 +1,51 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
+
+use crate::gecko_bindings::{bindings, structs::mozilla};
+use crate::json_writer::JSONWriter;
+use crate::marker::deserializer_tags_state::{
+    get_marker_type_functions_read_guard, MarkerTypeFunctions,
+};
+use std::ops::DerefMut;
+use std::os::raw::c_char;
+
+#[no_mangle]
+pub unsafe extern "C" fn gecko_profiler_serialize_marker_for_tag(
+    deserializer_tag: u8,
+    payload: *const u8,
+    payload_size: usize,
+    json_writer: &mut mozilla::baseprofiler::SpliceableJSONWriter,
+) {
+    let marker_type_functions = get_marker_type_functions_read_guard();
+    let &MarkerTypeFunctions {
+        transmute_and_stream_fn,
+        marker_type_name_fn,
+        ..
+    } = marker_type_functions.get(deserializer_tag);
+    let mut json_writer = JSONWriter::new(&mut *json_writer);
+
+    // Serialize the marker type name first.
+    json_writer.string_property("type", marker_type_name_fn());
+    // Serialize the marker payload now.
+    transmute_and_stream_fn(payload, payload_size, &mut json_writer);
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn gecko_profiler_stream_marker_schemas(
+    json_writer: &mut mozilla::baseprofiler::SpliceableJSONWriter,
+) {
+    let marker_type_functions = get_marker_type_functions_read_guard();
+
+    for funcs in marker_type_functions.iter() {
+        let marker_name = (funcs.marker_type_name_fn)();
+        let mut marker_schema = (funcs.marker_type_display_fn)();
+
+        bindings::gecko_profiler_marker_schema_stream(
+            json_writer,
+            marker_name.as_ptr() as *const c_char,
+            marker_name.len(),
+            marker_schema.pin.deref_mut().as_mut_ptr(),
+        )
+    }
+}

tools/profiler/rust-api/src/gecko_bindings/mod.rs

@@ -17,4 +17,5 @@ pub mod structs {
 
 pub use self::structs as bindings;
 
+mod glue;
 pub mod profiling_categories;

tools/profiler/rust-api/src/marker/deserializer_tags_state.rs

@@ -6,7 +6,7 @@ use crate::json_writer::JSONWriter;
 use crate::marker::schema::MarkerSchema;
 use crate::marker::{transmute_and_stream, ProfilerMarker};
 use std::collections::HashMap;
-use std::sync::RwLock;
+use std::sync::{RwLock, RwLockReadGuard};
 
 lazy_static! {
     static ref DESERIALIZER_TAGS_STATE: RwLock<DeserializerTagsState> =
@@ -89,3 +89,28 @@ where
         Some(deserializer_tag) => *deserializer_tag,
     }
 }
+
+/// A guard that will be used by the marker FFI functions for getting marker type functions.
+pub struct MarkerTypeFunctionsReadGuard {
+    guard: RwLockReadGuard<'static, DeserializerTagsState>,
+}
+
+impl MarkerTypeFunctionsReadGuard {
+    pub fn iter<'a>(&'a self) -> impl Iterator<Item = &'a MarkerTypeFunctions> {
+        self.guard.marker_type_functions_1_based.iter()
+    }
+
+    pub fn get<'a>(&'a self, deserializer_tag: u8) -> &'a MarkerTypeFunctions {
+        self.guard
+            .marker_type_functions_1_based
+            .get(deserializer_tag as usize - 1)
+            .expect("Failed to find the marker type functions for given deserializer tag")
+    }
+}
+
+/// Locks the DESERIALIZER_TAGS_STATE and returns the marker type functions read guard.
+pub fn get_marker_type_functions_read_guard() -> MarkerTypeFunctionsReadGuard {
+    MarkerTypeFunctionsReadGuard {
+        guard: DESERIALIZER_TAGS_STATE.read().unwrap(),
+    }
+}